141 lines
6.2 KiB
Python
141 lines
6.2 KiB
Python
# -*- coding: utf-8 -*-
|
|
import os
|
|
import sys
|
|
import logging
|
|
from qcloud_cos import CosConfig
|
|
from qcloud_cos import CosS3Client
|
|
from qcloud_cos.cos_exception import CosServiceError
|
|
|
|
# 配置日志输出格式（同时输出到屏幕和文件）
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(levelname)s - %(message)s',
|
|
handlers=[
|
|
logging.FileHandler("cos_upload.log", mode='a'), # 追加模式保存到日志文件
|
|
logging.StreamHandler(sys.stdout) # 在终端屏幕显示
|
|
]
|
|
)
|
|
|
|
# ==========================================
|
|
# 必填：请在此处替换为您自己的 COS 配置信息
|
|
# ==========================================
|
|
SECRET_ID = 'AKIDZ0EWnsmWnwvmT0KRLFC4mNlyzufDYRhm' # 替换为您的 SecretId
|
|
SECRET_KEY = 'yjJiuseHhuCLK8mcVVbaRJAwfHrKya0C' # 替换为您的 SecretKey
|
|
REGION = 'ap-guangzhou' # 替换为您存储桶所在的地域，例如 ap-guangzhou
|
|
BUCKET = 'gz-1349751149' # 替换为您的 Bucket 名称，包含 appid
|
|
# ==========================================
|
|
|
|
# 选填：高级配置
|
|
UPLOAD_DIR = '.' # 要上传的本地目录，'.' 代表当前目录
|
|
COS_PREFIX = 'uploads/images/20260513/data/' # 上传到 COS 时的远程路径前缀（如 'my_folder/'），留空则存放在根目录
|
|
IGNORE_FILES = ['.git', '.env', '__pycache__'] # 忽略的文件或目录片段
|
|
|
|
def init_cos_client():
|
|
"""初始化 COS 客户端"""
|
|
# 针对超大文件，增加 Timeout 超时时间（单位：秒），防止网络波动导致断开
|
|
config = CosConfig(Region=REGION, SecretId=SECRET_ID, SecretKey=SECRET_KEY, Timeout=300)
|
|
return CosS3Client(config)
|
|
|
|
def should_ignore(file_path):
|
|
"""判断是否需要忽略该文件"""
|
|
# 忽略脚本自身
|
|
if os.path.basename(file_path) == os.path.basename(__file__):
|
|
return True
|
|
|
|
# 忽略指定的隐藏文件或缓存目录
|
|
for ignore_item in IGNORE_FILES:
|
|
if ignore_item in file_path:
|
|
return True
|
|
return False
|
|
|
|
class UploadProgress:
|
|
def __init__(self, filename):
|
|
self.filename = os.path.basename(filename)
|
|
self.reported_percents = set()
|
|
|
|
def __call__(self, consumed_bytes, total_bytes):
|
|
if total_bytes:
|
|
rate = int(100 * (float(consumed_bytes) / float(total_bytes)))
|
|
# 每隔 10% 打印一次日志，避免日志过多刷屏
|
|
if rate % 10 == 0 and rate not in self.reported_percents:
|
|
self.reported_percents.add(rate)
|
|
logging.info("[PROGRESS] %s 进度: %d%% (%d/%d 字节)", self.filename, rate, consumed_bytes, total_bytes)
|
|
|
|
def upload_files(client):
|
|
"""遍历目录并上传文件"""
|
|
success_count = 0
|
|
fail_count = 0
|
|
|
|
# 遍历本地目录
|
|
for root, dirs, files in os.walk(UPLOAD_DIR):
|
|
for file in files:
|
|
local_file_path = os.path.join(root, file)
|
|
|
|
# 过滤不需要上传的文件
|
|
if should_ignore(local_file_path):
|
|
continue
|
|
|
|
# 计算在 COS 上的对象键 (Object Key)，即远程相对路径
|
|
rel_path = os.path.relpath(local_file_path, UPLOAD_DIR)
|
|
|
|
# 兼容 Windows 路径：将反斜杠转为正斜杠
|
|
rel_path_unix = rel_path.replace('\\', '/')
|
|
|
|
# 拼接 COS 远程前缀
|
|
if COS_PREFIX:
|
|
cos_key = "{0}/{1}".format(COS_PREFIX.rstrip('/'), rel_path_unix)
|
|
else:
|
|
cos_key = rel_path_unix
|
|
|
|
try:
|
|
# ==========================================
|
|
# 不重复上传逻辑：检查远端是否存在且大小一致
|
|
# ==========================================
|
|
local_size = os.path.getsize(local_file_path)
|
|
try:
|
|
# 获取 COS 上的文件信息
|
|
head = client.head_object(Bucket=BUCKET, Key=cos_key)
|
|
remote_size = int(head.get('Content-Length', 0))
|
|
|
|
# 如果远端文件存在且大小一致，则认为已上传过，直接跳过
|
|
if remote_size == local_size:
|
|
logging.info("[SKIP] 远端已存在且大小一致，跳过上传: {0}".format(cos_key))
|
|
success_count += 1
|
|
continue
|
|
except CosServiceError as e:
|
|
# 404 说明远端文件不存在，可以正常开始上传；其他错误则抛出
|
|
if e.get_status_code() != 404:
|
|
raise e
|
|
|
|
logging.info("[START] 正在上传: {0} -> cos://{1}/{2}".format(local_file_path, BUCKET, cos_key))
|
|
|
|
response = client.upload_file(
|
|
Bucket=BUCKET,
|
|
LocalFilePath=local_file_path,
|
|
Key=cos_key,
|
|
# 80G超大文件优化：增加单个分块大小，减小并发防止带宽被占满而导致超时
|
|
PartSize=100, # 分块大小改为 100 MB
|
|
MAXThread=3, # 并发线程数降低至 3，求稳
|
|
EnableMD5=False, # 是否校验 MD5
|
|
progress_callback=UploadProgress(local_file_path) # 添加进度回调
|
|
)
|
|
logging.info("[SUCCESS] 上传成功: {0}".format(cos_key))
|
|
success_count += 1
|
|
except Exception as e:
|
|
logging.error("[ERROR] 上传失败: {0}, 错误信息: {1}".format(local_file_path, e))
|
|
fail_count += 1
|
|
|
|
logging.info("[DONE] 全部上传任务执行完毕！成功: {0} 个文件，失败: {1} 个文件。".format(success_count, fail_count))
|
|
|
|
if __name__ == '__main__':
|
|
# 简单拦截，防止未修改配置直接运行
|
|
if SECRET_ID == 'YOUR_SECRET_ID':
|
|
logging.error("运行失败：请先在脚本代码中配置您的 SECRET_ID, SECRET_KEY, REGION 和 BUCKET 信息。")
|
|
sys.exit(1)
|
|
|
|
logging.info("初始化 COS 客户端...")
|
|
client = init_cos_client()
|
|
|
|
logging.info("开始扫描当前目录并准备上传...")
|
|
upload_files(client)
|