更新

2026-07-16 12:11:05 +08:00
parent 22ec32eb29
commit 32895b1591
41 changed files with 5740 additions and 0 deletions
@@ -0,0 +1,140 @@
+# -*- coding: utf-8 -*-
+import os
+import sys
+import logging
+from qcloud_cos import CosConfig
+from qcloud_cos import CosS3Client
+from qcloud_cos.cos_exception import CosServiceError
+
+# 配置日志输出格式（同时输出到屏幕和文件）
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler("cos_upload.log", mode='a'), # 追加模式保存到日志文件
+        logging.StreamHandler(sys.stdout)                # 在终端屏幕显示
+    ]
+)
+
+# ==========================================
+# 必填：请在此处替换为您自己的 COS 配置信息
+# ==========================================
+SECRET_ID = 'AKIDZ0EWnsmWnwvmT0KRLFC4mNlyzufDYRhm'      # 替换为您的 SecretId
+SECRET_KEY = 'yjJiuseHhuCLK8mcVVbaRJAwfHrKya0C'    # 替换为您的 SecretKey
+REGION = 'ap-guangzhou'           # 替换为您存储桶所在的地域，例如 ap-guangzhou
+BUCKET = 'gz-1349751149' # 替换为您的 Bucket 名称，包含 appid
+# ==========================================
+
+# 选填：高级配置
+UPLOAD_DIR = '.'                  # 要上传的本地目录，'.' 代表当前目录
+COS_PREFIX = 'uploads/images/20260513/data/'                   # 上传到 COS 时的远程路径前缀（如 'my_folder/'），留空则存放在根目录
+IGNORE_FILES = ['.git', '.env', '__pycache__'] # 忽略的文件或目录片段
+
+def init_cos_client():
+    """初始化 COS 客户端"""
+    # 针对超大文件，增加 Timeout 超时时间（单位：秒），防止网络波动导致断开
+    config = CosConfig(Region=REGION, SecretId=SECRET_ID, SecretKey=SECRET_KEY, Timeout=300)
+    return CosS3Client(config)
+
+def should_ignore(file_path):
+    """判断是否需要忽略该文件"""
+    # 忽略脚本自身
+    if os.path.basename(file_path) == os.path.basename(__file__):
+        return True
+    
+    # 忽略指定的隐藏文件或缓存目录
+    for ignore_item in IGNORE_FILES:
+        if ignore_item in file_path:
+            return True
+    return False
+
+class UploadProgress:
+    def __init__(self, filename):
+        self.filename = os.path.basename(filename)
+        self.reported_percents = set()
+
+    def __call__(self, consumed_bytes, total_bytes):
+        if total_bytes:
+            rate = int(100 * (float(consumed_bytes) / float(total_bytes)))
+            # 每隔 10% 打印一次日志，避免日志过多刷屏
+            if rate % 10 == 0 and rate not in self.reported_percents:
+                self.reported_percents.add(rate)
+                logging.info("[PROGRESS] %s 进度: %d%% (%d/%d 字节)", self.filename, rate, consumed_bytes, total_bytes)
+
+def upload_files(client):
+    """遍历目录并上传文件"""
+    success_count = 0
+    fail_count = 0
+    
+    # 遍历本地目录
+    for root, dirs, files in os.walk(UPLOAD_DIR):
+        for file in files:
+            local_file_path = os.path.join(root, file)
+            
+            # 过滤不需要上传的文件
+            if should_ignore(local_file_path):
+                continue
+            
+            # 计算在 COS 上的对象键 (Object Key)，即远程相对路径
+            rel_path = os.path.relpath(local_file_path, UPLOAD_DIR)
+            
+            # 兼容 Windows 路径：将反斜杠转为正斜杠
+            rel_path_unix = rel_path.replace('\\', '/')
+            
+            # 拼接 COS 远程前缀
+            if COS_PREFIX:
+                cos_key = "{0}/{1}".format(COS_PREFIX.rstrip('/'), rel_path_unix)
+            else:
+                cos_key = rel_path_unix
+                
+            try:
+                # ==========================================
+                # 不重复上传逻辑：检查远端是否存在且大小一致
+                # ==========================================
+                local_size = os.path.getsize(local_file_path)
+                try:
+                    # 获取 COS 上的文件信息
+                    head = client.head_object(Bucket=BUCKET, Key=cos_key)
+                    remote_size = int(head.get('Content-Length', 0))
+                    
+                    # 如果远端文件存在且大小一致，则认为已上传过，直接跳过
+                    if remote_size == local_size:
+                        logging.info("[SKIP] 远端已存在且大小一致，跳过上传: {0}".format(cos_key))
+                        success_count += 1
+                        continue
+                except CosServiceError as e:
+                    # 404 说明远端文件不存在，可以正常开始上传；其他错误则抛出
+                    if e.get_status_code() != 404:
+                        raise e
+
+                logging.info("[START] 正在上传: {0} -> cos://{1}/{2}".format(local_file_path, BUCKET, cos_key))
+                
+                response = client.upload_file(
+                    Bucket=BUCKET,
+                    LocalFilePath=local_file_path,
+                    Key=cos_key,
+                    # 80G超大文件优化：增加单个分块大小，减小并发防止带宽被占满而导致超时
+                    PartSize=100,    # 分块大小改为 100 MB
+                    MAXThread=3,     # 并发线程数降低至 3，求稳
+                    EnableMD5=False, # 是否校验 MD5
+                    progress_callback=UploadProgress(local_file_path) # 添加进度回调
+                )
+                logging.info("[SUCCESS] 上传成功: {0}".format(cos_key))
+                success_count += 1
+            except Exception as e:
+                logging.error("[ERROR] 上传失败: {0}, 错误信息: {1}".format(local_file_path, e))
+                fail_count += 1
+                
+    logging.info("[DONE] 全部上传任务执行完毕！成功: {0} 个文件，失败: {1} 个文件。".format(success_count, fail_count))
+
+if __name__ == '__main__':
+    # 简单拦截，防止未修改配置直接运行
+    if SECRET_ID == 'YOUR_SECRET_ID':
+        logging.error("运行失败：请先在脚本代码中配置您的 SECRET_ID, SECRET_KEY, REGION 和 BUCKET 信息。")
+        sys.exit(1)
+        
+    logging.info("初始化 COS 客户端...")
+    client = init_cos_client()
+    
+    logging.info("开始扫描当前目录并准备上传...")
+    upload_files(client)