import filecmp
comparison = filecmp.dircmp(dir_local, dir_server)
comparison.report_full_closure()
我希望将本地机器上保存的所有CSV文件与服务器上保存的文件进行比较。两者的文件夹结构相同。我只想进行数据比较,而不是元数据(如创建时间等)。我正在使用filecmp,但它似乎执行元数据比较。是否有方法可以做到我想要的?
比较服务器文件系统和本地文件系统之间的.csv文件有多种方法。
方法一:使用哈希值
此方法使用Python模块hashlib。我使用哈希算法sha256来计算文件的哈希摘要。我将比较具有相同文件名的文件的哈希值。这种方法效果很好,但会忽略任何在两个目录中不存在的文件。
import hashlib
def compare_common_files_by_hash(directory_one, directory_two):
d1_files = set(os.listdir(directory_one))
d2_files = set(os.listdir(directory_two))
common_files = list(d1_files & d2_files)
if common_files:
for filename in common_files:
hash_01 = hashlib.sha256(open(f'{directory_one}/{filename}', 'rb').read()).hexdigest()
hash_02 = hashlib.sha256(open(f'{directory_two}/{filename}', 'rb').read()).hexdigest()
if hash_01 == hash_02:
print(f'The file - {filename} is identical in the directories {directory_one} and {directory_two}')
elif hash_01 != hash_02:
print(f'The file - {filename} is different in the directories {directory_one} and {directory_two}')
方法二:使用os的st_size
这种方法使用了Python模块os。在此示例中,我比较了文件的大小。这种方法可以正常工作,但是它会将任何数据更改不会改变文件大小的文件错误地分类。
import os
def compare_common_files_by_size(directory_one, directory_two):
d1_files = set(os.listdir(directory_one))
d2_files = set(os.listdir(directory_two))
common_files = list(d1_files & d2_files)
if common_files:
for filename in common_files:
file_01 = os.stat(f'{directory_one}/{filename}')
file_02 = os.stat(f'{directory_two}/{filename}')
if file_01.st_size == file_02.st_size:
print(f'The file - {filename} is identical in the directories {directory_one} and {directory_two}')
elif file_01.st_size != file_02.st_size:
print(f'The file - {filename} is different in the directories {directory_one} and'
f' {directory_two}')
第三种方法:使用os.st_size和os.st_mtime
这种方法同样需要使用Python模块os。在这个例子中,我不仅比较了文件大小,还比较了最后修改时间。这种方法效果很好,但是它也会错误地将一些文件分类为相同的。在测试中,我保存了一个没有数据修改的文件,但os.st_mtime标记该文件为不同的,但实际上文件并没有真正不同。
import os
def compare_common_files_by_metadata(directory_one, directory_two):
d1_files = set(os.listdir(directory_one))
d2_files = set(os.listdir(directory_two))
common_files = list(d1_files & d2_files)
if common_files:
for filename in common_files:
file_01 = os.stat(f'{directory_one}/{filename}')
file_02 = os.stat(f'{directory_two}/{filename}')
if file_01.st_size == file_02.st_size and file_01.st_mtime == file_02.st_mtime:
print(f'The file - {filename} is identical in the directories {directory_one} and {directory_two}')
elif file_01.st_size != file_02.st_size or file_01.st_mtime != file_02.st_mtime:
print(f'The file - {filename} is different in the directories {directory_one} and'
f' {directory_two}')
方法四:使用 set()
这个示例使用 Python 的 set() 方法来确定两个同名 csv 文件之间的行差异。这种方法将输出两个 csv 文件之间的确切更改。
import os
def compare_common_files_by_lines(directory_one, directory_two):
d1_files = set(os.listdir(directory_one))
d2_files = set(os.listdir(directory_two))
common_files = list(d1_files & d2_files)
if common_files:
for filename in common_files:
if fileName.endswith('.csv'):
file_01 = open(f'{directory_one}/{filename}', 'r', encoding='ISO-8859-1')
file_02 = open(f'{directory_two}/{filename}', 'r', encoding='ISO-8859-1')
csv_file_01 = set(map(tuple, csv.reader(file_01)))
csv_file_02 = set(map(tuple, csv.reader(file_02)))
different = csv_file_01 ^ csv_file_02
for row in sorted(different, key=lambda x: x, reverse=True):
if row:
print(f'This row: \n {row} \n was different between the file {fileName} in the directories'
f' {directory_one} and {directory_two}')
第五种方法:使用 filecmp.cmp
这种方法使用了 Python 模块 filecmp. 在这个例子中,我使用了 shallow 参数设置为 False 的 filecmp.cmp。将此参数设置为 False 就会让 filecmp 查看文件内容而不是元数据(例如文件大小),这是 filecmp.cmp 的默认设置。这种方法和使用 hashlib 的第一种方法一样有效。
import filecmp
def compare_common_files(directory_one, directory_two):
d1_files = set(os.listdir(directory_one))
d2_files = set(os.listdir(directory_two))
common_files = list(d1_files & d2_files)
if common_files:
for filename in common_files:
file_01 = f'{directory_one}/{filename}'
file_02 = f'{directory_two}/{filename}'
comparison = filecmp.cmp(file_01, file_02, shallow=False)
if comparison:
print(f'The file - {filename} is identical in the directories - {directory_one} and {directory_two}')
elif not comparison:
print(f'The file - {filename} is different in the directories - {directory_one} and {directory_two}')
方法六:使用filecmp.dircmp
这种方法也使用了Python模块filecmp。 在这个例子中,我使用了filecmp.dircmp,它不仅可以识别两个目录之间不共同的文件,并查找那些具有相似名称但内容不同的文件。
import filecmp
def directory_recursive(directory_one, directory_two):
files = filecmp.dircmp(directory_one, directory_two)
for filename in files.diff_files:
print(f'The file - {filename} is different in the directories - {files.left} and {files.right}')
for filename in files.left_only:
print(f'The file - {filename} - was only found in the directory {files.left}')
for filename in files.right_only:
print(f'The file - {filename} - was only found in the directory {files.right}')
方法七:逐行比较
该示例对两个CSV文件进行逐行比较,并输出不同的行。输出结果可以添加到Python字典或JSON文件中以备后用。
import csv
def get_csv_file_lines(file):
with open(file, 'r', encoding='utf-8') as csv_file:
rows = csv.reader(csv_file)
for row in rows:
yield row
def compare_csv_files_line_by_line(csv_file_one, csv_file_two):
csvfile_02 = get_csv_file_lines(csv_file_two)
for line_one in get_csv_file_lines(csv_file_one):
line_two = csvfile_02.__next__()
if line_two != line_one:
print('File names being compared:')
print(f'csv_file_one: {csv_file_one}')
print(f'csv_file_two: {csv_file_two}')
print(f'The following rows have difference in the files being compared.')
print('csv_file_one:', line_one)
print('csv_file_two:', line_two)
print('\n')
使用hashlib实现本地文件系统到S3存储桶的转换
下面的示例是一个实际的用例,用于在本地文件系统和远程S3存储桶之间比较文件。我最初打算使用AWS S3创建的object.e_tag,但该标记可能存在问题,并且不应在哈希比较操作中使用。我决定查询S3并将单个文件加载到可查询和清空的内存文件系统中,在每次比较操作期间使用该方法非常有效,并且对我的系统性能没有不良影响。
import fs
import os
import boto3
import hashlib
def create_temp_memory_filesystem():
mem_fs = fs.open_fs('mem://')
virtual_disk = mem_fs.makedir('hidden_dir')
return mem_fs, virtual_disk
def query_s3_file_by_name(filename, memory_filesystem, temp_directory):
s3 = boto3.resource('s3', aws_access_key_id='your_access_key_id',
aws_secret_access_key='your_secret_access_key')
bucket = s3.Bucket('your_bucket_name')
for obj in bucket.objects.all():
if obj.key == filename:
body = obj.get()['Body'].read()
with memory_filesystem.open(f'{temp_directory}/s3_{filename}', 'w') as f:
f.write(str(body))
f.close()
def compare_local_files_to_s3_files(local_csv_files):
virtual_disk = create_temp_memory_filesystem()
directory_name = str(virtual_disk[1]).split('/')[1]
files = set(os.listdir(local_csv_files))
for filename in files:
if filename.endswith('.csv'):
local_file_hash = hashlib.sha256(open(f'{local_csv_files}/{filename}', 'rb').read()).hexdigest()
query_s3_file_by_name(filename, virtual_disk[0], directory_name)
virtual_files = virtual_disk[0].opendir(directory_name)
for file_name in virtual_files.listdir('/'):
s3_file_hash = hashlib.sha256(open(file_name, 'rb').read()).hexdigest()
if local_file_hash == s3_file_hash:
print(f'The file - {filename} is identical in both the local file system and the S3 bucket.')
elif local_file_hash != s3_file_hash:
print(f'The file - {filename} is different between the local file system and the S3 bucket.')
virtual_files.remove(file_name)
virtual_disk[0].close()
使用filecmp将本地文件系统转移到S3存储桶
这个示例与上面的示例相同,只是我使用filecmp.cmp而不是hashlib进行比较操作。
import fs
import os
import boto3
import filecmp
def create_temp_memory_filesystem():
mem_fs = fs.open_fs('mem://')
virtual_disk = mem_fs.makedir('hidden_dir')
return mem_fs, virtual_disk
def query_s3_file_by_name(filename, memory_filesystem, temp_directory):
s3 = boto3.resource('s3', aws_access_key_id='your_access_key_id',
aws_secret_access_key='your_secret_access_key')
bucket = s3.Bucket('your_bucket_name')
for obj in bucket.objects.all():
if obj.key == filename:
body = obj.get()['Body'].read()
with memory_filesystem.open(f'{temp_directory}/s3_{filename}', 'w') as f:
f.write(str(body))
f.close()
def compare_local_files_to_s3_files(local_csv_files):
virtual_disk = create_temp_memory_filesystem()
directory_name = str(virtual_disk[1]).split('/')[1]
files = set(os.listdir(local_csv_files))
for filename in files:
if filename.endswith('.csv'):
local_file = f'{local_csv_files}/{filename}'
query_s3_file_by_name(filename, virtual_disk[0], directory_name)
virtual_files = virtual_disk[0].opendir(directory_name)
for file_name in virtual_files.listdir('/'):
comparison = filecmp.cmp(local_file, file_name, shallow=False)
if comparison:
print(f'The file - {filename} is identical in both the local file system and the S3 bucket.')
elif not comparison:
print(f'The file - {filename} is different between the local file system and the S3 bucket.')
virtual_files.remove(file_name)
virtual_disk[0].close()
使用hashlib将本地文件系统上传到Google Cloud存储桶
这个例子和上面的S3 hashlib代码示例类似,但它使用了一个Google Cloud存储桶。
import fs
import os
import hashlib
from google.cloud import storage
def create_temp_memory_filesystem():
mem_fs = fs.open_fs('mem://')
virtual_disk = mem_fs.makedir('hidden_dir')
return mem_fs, virtual_disk
def query_google_cloud_storage_file_by_name(filename, memory_filesystem, temp_directory):
client = storage.Client.from_service_account_json('path_to_your_credentials.json')
bucket = client.get_bucket('your_bucket_name')
blobs = bucket.list_blobs()
for blob in blobs:
if blob.name == filename:
with memory_filesystem.open(f'{temp_directory}/{filename}', 'w') as f:
f.write(str(blob.download_to_filename(blob.name)))
f.close()
def compare_local_files_to_google_storage_files(local_csv_files):
virtual_disk = create_temp_memory_filesystem()
directory_name = str(virtual_disk[1]).split('/')[1]
files = set(os.listdir(local_csv_files))
for filename in files:
if filename.endswith('.csv'):
local_file_hash = hashlib.sha256(open(f'{local_csv_files}/{filename}', 'rb').read()).hexdigest()
query_google_cloud_storage_file_by_name(filename, virtual_disk[0], directory_name)
virtual_files = virtual_disk[0].opendir(directory_name)
for file_name in virtual_files.listdir('/'):
gs_file_hash = hashlib.sha256(open(file_name, 'rb').read()).hexdigest()
if local_file_hash == gs_file_hash:
print(f'The file - {filename} is identical in both the local file system and the Google Cloud bucket.')
elif local_file_hash != gs_file_hash:
print(f'The file - {filename} is different between the local file system and the Google Cloud bucket.')
virtual_files.remove(file_name)
virtual_disk[0].close()
使用 filecmp 将本地文件系统复制到 Google Cloud 存储桶
这个示例与上面的 S3 filecmp 代码示例类似,但它使用了一个 Google Cloud 存储桶。
import fs
import os
import filecmp
from google.cloud import storage
def create_temp_memory_filesystem():
mem_fs = fs.open_fs('mem://')
virtual_disk = mem_fs.makedir('hidden_dir')
return mem_fs, virtual_disk
def query_google_cloud_storage_file_by_name(filename, memory_filesystem, temp_directory):
client = storage.Client.from_service_account_json('path_to_your_credentials.json')
bucket = client.get_bucket('your_bucket_name')
blobs = bucket.list_blobs()
for blob in blobs:
if blob.name == filename:
with memory_filesystem.open(f'{temp_directory}/{filename}', 'w') as f:
f.write(str(blob.download_to_filename(blob.name)))
f.close()
def compare_local_files_to_google_storage_files(local_csv_files):
virtual_disk = create_temp_memory_filesystem()
directory_name = str(virtual_disk[1]).split('/')[1]
files = set(os.listdir(local_csv_files))
for filename in files:
if filename.endswith('.csv'):
local_file = f'{local_csv_files}/{filename}'
query_google_cloud_storage_file_by_name(filename, virtual_disk[0], directory_name)
virtual_files = virtual_disk[0].opendir(directory_name)
for file_name in virtual_files.listdir('/'):
comparison = filecmp.cmp(local_file, file_name, shallow=False)
if comparison:
print(f'The file - {filename} is identical in both the local file system and the Google Cloud bucket.')
elif not comparison:
print(f'The file - {filename} is different between the local file system and the Google Cloud bucket.')
virtual_files.remove(file_name)
virtual_disk[0].close()
使用可选的 shallow
参数来调用 filecmp
:一个布尔值,可以是 'True' 或 'False'。该参数的默认值为 True。
根据文档:
如果 shallow 为 true,并且两个文件的 os.stat() 标志(文件类型、大小和修改时间)相同,则认为这些文件相等。
否则,如果它们的大小或内容不同,则将这些文件视为不同。
import filecmp
# Path of first file
file1 = "/home/geeks/Desktop/gfg/data.txt"
# Path of second file
file2 = "/home/geeks/Desktop/gfg/gfg.txt"
# Compare the os.stat()
# signature i.e the metadata
# of both files
comp = filecmp.cmp(file1, file2)
# Print the result of comparison
print(comp)
# Compare the
# contents of both files
comp = filecmp.cmp(file1, file2, shallow = False)
# Print the result of comparison
print(comp)
文档 链接
(注:此翻译涉及技术内容,如有不准确之处,恳请指正)filecmp.dircmp
执行的是浅层次比较:
浅层次比较意味着dircmp类通过执行如filecmp.cmp()所述的浅层次比较来比较文件。
filecmp
将检查文件A和文件B的os.stat是否相等。如果相等,则返回true。如果不相等,则比较A和B的内容,如果它们相等,则返回true,否则返回false。filecmp.cmpfiles(dir1, dir2, common, shallow=False)
。请注意,filecmp.cmpfiles
的工作方式如下:
您可以在此处阅读更多信息:此处。比较由common给出的两个目录dir1和dir2中的文件。
filecmp.cmp(f1, f2, shallow=False)
。您可以在此处阅读有关filecmp.cmp
的更多信息。根据filecmp
文档:
filecmp模块定义了用于比较文件和目录的函数,具有各种可选的时间/正确性权衡。有关比较文件的内容,请参见difflib模块。
具体而言,.cmp
和.cmpfiles
使用它们的签名以及其他元数据来比较文件:
filecmp.cmp(f1, f2, shallow=True)
比较名为f1和f2的文件,如果它们似乎相等,则返回True,否则返回False。 如果shallow为True,则将具有相同os.stat()签名的文件视为相等。否则,将比较文件的内容。 请注意,此函数不调用任何外部程序,因此具有可移植性和效率。 此函数对过去的比较和结果使用缓存,如果文件的os.stat()信息发生更改,则缓存条目无效。可以使用clear_cache()清除整个缓存。
比较两个目录dir1和dir2中名称相同的文件。返回三个文件名列表:匹配、不匹配和错误。匹配包含匹配的文件列表,不匹配包含不匹配的文件名,错误列出了无法比较的文件名。如果文件不存在于其中一个目录中,用户缺少读取权限或由于其他原因无法进行比较,则将文件列在错误中。shallow参数具有与filecmp.cmp()相同的含义和默认值。例如,cmpfiles('a', 'b', ['c', 'd/e'])将比较a/c和b/c以及a/d/e和b/d/e。'c'和'd/e'将分别在三个返回的列表中。此外,如果您希望获得差异输出,请考虑使用filecmp文档中提到的difflib。参考资料:filecmp: https://docs.python.org/3/library/filecmp.htmlfilecmp.cmpfiles(dir1, dir2, common, shallow=True)
difflib:https://docs.python.org/3/library/difflib.html#module-difflib(差异比较库)
diff
;如果你在Windows上,请调用fc
。也就是说,如果你只想知道它们是否具有相同的数据。你需要能够本地访问“从服务器”下载的文件,所以要么在比较后下载并删除它们,要么挂载共享驱动器(如果可行)。由于你将在代码运行的位置比较数据,因此你必须上传你的数据或下载服务器数据,所以在完成后将其删除。
例如,在Windows上:import subprocess
def files_are_a_match(file1, file2):
"""function returns True for matching files. False on mismatch or error. Expects windows file paths as strings"""
try:
cmd = f"fc /B \"{file1}\" \"{file2}\""
txt = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
except:
return False
return "FC: no differences encountered" in str(txt)
https://gitpython.readthedocs.io/en/stable/
from git import Repo
repo = Repo('my_repo')
# Check differences between current files and last commit
diff = repo.git.diff(repo.head.commit.tree)
print(diff)
以下是一种比较CSV文件内容的方法:
导入包并创建两个函数:
import hashlib
from pathlib import Path
from time import perf_counter
def sha256sum(filename):
''' source: https://dev59.com/uGEh5IYBdhLWcg3wwlu8#44873382 '''
h = hashlib.sha256()
b = bytearray(128 * 1024)
mv = memoryview(b)
with open(filename, 'rb', buffering=0) as f:
for n in iter(lambda : f.readinto(mv), 0):
h.update(mv[:n])
return h.hexdigest()
def csv_hashes(dir_name):
''' Map CSV filenames to SHA hashes. '''
return { csv_file: sha256sum(csv_file)
for csv_file in dir_name.rglob('*.csv') }
local_dir = Path('../../../projects')
start = perf_counter()
local_hashes = csv_hashes(local_dir)
elapsed = perf_counter() - start
rate = len(local_hashes) / elapsed
print(f'indexed {rate:.3f} files/sec')
indexed 53.342 files/sec ## too slow for real-world use case?
在远程机器上重复操作,并比较这两个字典。
#!/usr/bin/env python3
import pysftp
import sys
from pathlib import Path
from io import BytesIO
import re
LOCAL_DIR = 'C:\\My\\Directory\\' # with closing separator
REMOTE_DIR = '/home/directory/' # absolute directory with closing separator
class Sftp:
def __init__(self, host, port, username, password, deploymentDirectory, verbose=True):
if deploymentDirectory[-1] != '/': deploymentDirectory += '/'
self.deployment_directory = deploymentDirectory
self.verbose = verbose
self.connection = None
try:
self.connection = pysftp.Connection(host, port=port, username=username, password=password)
except Exception:
print('Could not connect to remote sftp server with the specified arguments.', file=sys.stderr)
sys.exit(1)
def __del__(self):
self.close()
def close(self):
if self.connection:
self.connection.close()
self.connection = None
def read_text_file(self, remote_file_name):
full_remote_file_name = self.deployment_directory + remote_file_name
b = BytesIO()
self.connection.getfo(full_remote_file_name, b)
s = b.getvalue().decode('utf-8')
return s
def remote_file_exists(self, remote_file_name):
full_remote_file_name = self.deployment_directory + remote_file_name
return self.connection.isfile(full_remote_file_name)
def compare(local_text, remote_text):
"""
The files could be the same except for the way the hosts handle the line-termination sequence (Windows: \r\n, Unix/Linux: \n, Mac: \r).
So, let's normalize:
"""
rex = re.compile(r'\r\n?')
local_text = rex.sub('\n', local_text)
remote_text = rex.sub('\n', remote_text)
return local_text == local_text
def main():
sftp = Sftp(host='demo.com', port=22, username='xxxx', password='xxxx', deploymentDirectory=REMOTE_DIR)
l_local_dir = len(LOCAL_DIR)
for path in Path(LOCAL_DIR).rglob('*.csv'):
dir, file_name = path.parent, path.name
# compute relative remote path:
remote_file_name = str(dir)[l_local_dir:].replace('\\', '/') + '/' + file_name
if not sftp.remote_file_exists(remote_file_name):
print(f'{path}: This file does not exist in remote directory.')
else:
remote_text = sftp.read_text_file(remote_file_name)
with path.open(encoding='utf-8') as f:
local_text = f.read()
if compare(local_text, remote_text):
print(f'{path} exits in the remote directory and matches.')
else:
print(f'{path} exits in the remote directory but does not match.')
sftp.close()
main()
difflib
吗? - l'L'l