利用 GitLab CI 同步组织内的各项目的代码

jetsung

利用 GitLab CI 同步组织内的各项目的代码

利用 GitLab CI 定时计划，每天将本组织名下的所有仓库，从指定的源站更新最新代码。

将创建一个组织
在组织名下，创建一个仓库
设置仓库 CI 的环境变量 USER_TOKEN：设置 -> CI/CD -> 变量 -> 添加变量
设置定时计划：构建 -> 流水线计划 -> 新建计划 -> 创建流水线计划

本组织下的各项目信息内容设置为以下格式开头

 ::org/repo

 # 或
 ::github@org/repo

 # 或
 ::gitcode.com@org/repo

该仓库下文件列表为：

.gitlab-ci.yml
.codesync.sh

.gitlab-ci.yml

default:
  image: alpine:latest

stages:
  - sync

variables        :
  GIT_DEPTH: 0
  GIT_STRATEGY: clone
  GIT_SUBMODULE_STRATEGY: recursive
  TZ: Asia/Shanghai

sync-code:
  stage: sync
  before_script: |
    apk update
    apk add curl git jq bash
  script: |
    bash ./.codesync.sh

.codesync.sh

#!/usr/bin/env bash
#
# Name: .codesync.sh
# Description: 从源站拉取代码，并更新到指定目标仓库
# Author: Jetsung Chan <jetsungchan@gmail.com>
#
#

set -euo pipefail

# 判断是否为 URL 的函数
is_url() {
    local url="$1"
    # 正则表达式匹配 URL
    if [[ "$url" =~ ^https?://[^[:space:]]+ ]]; then
        return 0  # 是 URL
    else
        return 1  # 不是 URL
    fi
}

get_source_data() {
  local file_path="${1:-}"
  if is_url "$file_path"; then
    DATA=$(curl -fsSL "$file_path")
  elif [ -f "$file_path" ]; then
    DATA=$(cat "$file_path")
  else
    printf "\n\033[31m[ERROR] %s is not a valid file or URL\033[0m\n" "$file_path"
    exit 0
  fi
}

progress_markdown() {
  MARKDOWN_DATA="${1:-}"
  if [ -z "$MARKDOWN_DATA" ]; then
    printf "\n\033[31m[ERROR] MARKDOWN_DATA is empty\033[0m\n"
    exit 0
  fi

  echo -e "$MARKDOWN_DATA" | sed 's# ##g' | while IFS="|" read -r _ ORIGIN_HUB ORIGIN_REPO TARGET_REPO STATUS; do
    CODE_HUB=$(echo "$ORIGIN_HUB" | cut -d '`' -f 2)
    CODE_REPO=$(echo "$ORIGIN_REPO" | cut -d '`' -f 2)
    TARGET_REPO=$(echo "$TARGET_REPO" | cut -d '`' -f 2)
    STATUS="${STATUS%%|*}"

    write_repo_file "$CODE_HUB" "$CODE_REPO" "$TARGET_REPO" "$STATUS"
  done
}

write_repo_file() {
    CODE_HUB="${1:-}"
    CODE_REPO="${2:-}"
    TARGET_REPO="${3:-}"
    STATUS="${4:-}"

    # 规范化 STATUS 变量：去除首尾空格，并转换为小写
    STATUS=$(echo "$STATUS" | tr '[:upper:]' '[:lower:]' | xargs)

    if [ -z "$CODE_HUB" ] || [ -z "$CODE_REPO" ] || [ -z "$TARGET_REPO" ] || [ -z "$STATUS" ]; then
      return
    fi

    if [ "$STATUS" != "true" ] && [ "$STATUS" != "false" ]; then
      return
    fi

    # 通过 API 提取最近更新 PUSH
    # 对比参数，以提取是否更新
    # github: curl -s "https://api.github.com/repos/idevsig/filetas/events" | jq -r 'map(select(.type == "PushEvent")) | .[0].created_at' | xargs -I {} date -d {} +%s
    # gitlab: curl -s "https://framagit.org/api/v4/projects/idev%2Ffiletas/events" | jq -r 'map(select(.action_name == "pushed to")) | .[0].created_at'
    # gitee:  curl -s "https://gitee.com/api/v5/repos/cencxc/open-vector/events" | jq -r 'map(select(.type == "PushEvent")) | .[0].created_at'
    # gitcode: curl -s -H "PRIVATE-TOKEN: $GITCODE_TOKEN" 'https://api.gitcode.com/api/v5/repos/idev/filetas/events' | jq -r '.events[] | select(.action_name == "pushed to") | .created_at' | head -n 1
    # gitcode: curl -s -H "PRIVATE-TOKEN: $GITCODE_TOKEN" 'https://api.gitcode.com/api/v5/repos/idev/filetas/events?filter=push' | jq -r '.events[] | select(.action_name == "pushed to") | .created_at' | head -n 1

    echo ""
    echo "CODE_HUB: $CODE_HUB"
    echo "CODE_REPO: $CODE_REPO"
    echo "TARGET_REPO: $TARGET_REPO"
    echo "STATUS: $STATUS"      

    # 检查 STATUS 是否为 "true"
    if [ "$STATUS" != "true" ]; then
      return
    fi

    cat >> "$REPO_LIST" <<EOF
$CODE_HUB|$CODE_REPO|$TARGET_REPO
EOF
}

get_repos() {
  if [ -z "$SOURCE_URL" ]; then # GitLab 项目 Wiki URL
    get_repos_from_wiki_home
  else
    if echo "$SOURCE_URL" | grep -q ','; then # 自定义文件后缀
      get_repos_from_wiki_git
    elif [[ "$SOURCE_URL" == *.md ]]; then # Markdown
      get_repos_from_markdown
    elif [[ "$SOURCE_URL" == *.git ]]; then # Git
      get_repos_from_wiki_git
    else
      get_repos_from_json # JSON
    fi
  fi
}

update_date_from_api() {
  # 提取协议部分
  protocol=$(echo "$SOURCE_URL" | sed -E 's#^(https?)://.*#\1#')
  if [ "$protocol" != "http" ] && [ "$protocol" != "https" ]; then
    printf "\n\033[31m[ERROR] protocol is not http or https\033[0m\n"
    exit 0
  fi

  # 提取 hostname 部分
  hostname=$(echo "$SOURCE_URL" | sed -E 's#^https?://([^/]+).*#\1#')
  if [ -z "$hostname" ]; then
    printf "\n\033[31m[ERROR] hostname is empty\033[0m\n"
    exit 0
  fi

  SERVER_HOST="$hostname"
}  

get_repos_api() {
  update_date_from_api

  case "$SOURCE_URL" in
      # 腾讯工峰
      *"git.code.tencent.com"*) 
        get_repos_from_api_tencent
      ;;

      # GitCode
      *"gitcode.com"*) 
        get_repos_from_api_gitcode
      ;;
      
      # GitLab 系列
      *) 
        get_repos_from_api_gitlab
      ;;
  esac  
}

get_repos_from_api_tencent() {
  local _DATA

  USER_NAME=$(curl -fsSL -H "PRIVATE-TOKEN: $SOURCE_TOKEN" "$protocol://$hostname/api/v3/user" | jq -r '.username')
  if [ -z "$USER_NAME" ]; then
    printf "\n\033[31m[ERROR] USER_NAME is empty\033[0m\n"
    exit 0
  fi
  
  _DATA=$(curl -fsSL -H "PRIVATE-TOKEN:$SOURCE_TOKEN" "$SOURCE_URL")
  echo "$_DATA" | jq -c '.[]' | while read -r row; do
    _full_path=$(echo "$row" | jq -r '.path_with_namespace')
    _description=$(echo "$row" | jq -r '.description')
    _git_url_to_repo=$(echo "$row" | jq -r '.https_url_to_repo')  

    _desc=$(echo "$_description" | head -n 1 | awk -F ' ' '{print $1}')
    if echo "$_desc" | grep  -v -q '^::'; then
      echo "skip: $_full_path"
      continue
    fi

    # ::org/repo
    # ::github@org/repo
    # ::gitcode.com@org/repo
    IFS='@' read -r CODE_HUB CODE_REPO <<< "${_desc//::/}"

    # default github
    if [ -z "$CODE_REPO" ]; then
      CODE_REPO="$CODE_HUB"
      CODE_HUB="github"
    fi   

    write_repo_file "$CODE_HUB" "$CODE_REPO" "$_git_url_to_repo" "true"
  done
}

get_repos_from_api_gitcode() {
  local _DATA

  USER_NAME=$(curl -fsSL -H "PRIVATE-TOKEN: $SOURCE_TOKEN" "$protocol://$hostname/api/v5/user" | jq -r '.login')
  if [ -z "$USER_NAME" ]; then
    printf "\n\033[31m[ERROR] USER_NAME is empty\033[0m\n"
    exit 0
  fi

  # api.gitcode.com -> gitcode.com
  SERVER_HOST="${hostname//api.gitcode.com/gitcode.com}"

  _DATA=$(curl -fsSL -H "PRIVATE-TOKEN:$SOURCE_TOKEN" "$SOURCE_URL")
  echo "$_DATA" | jq -c '.[]' | while read -r row; do
    _full_path=$(echo "$row" | jq -r '.path')
    _description=$(echo "$row" | jq -r '.description')
    _git_url_to_repo=$(echo "$row" | jq -r '.html_url')
    _git_url_to_repo="$_git_url_to_repo.git"

    _desc=$(echo "$_description" | head -n 1 | awk -F ' ' '{print $1}')
    if echo "$_desc" | grep  -v -q '^::'; then
      echo "skip: $_full_path"
      continue
    fi

    # ::org/repo
    # ::github@org/repo
    # ::gitcode.com@org/repo
    IFS='@' read -r CODE_HUB CODE_REPO <<< "${_desc//::/}"

    # default github
    if [ -z "$CODE_REPO" ]; then
      CODE_REPO="$CODE_HUB"
      CODE_HUB="github"
    fi   

    write_repo_file "$CODE_HUB" "$CODE_REPO" "$_git_url_to_repo" "true"
  done
}

get_repos_from_api_gitlab() {
  local _DATA

  USER_NAME=$(curl -fsSL -H "PRIVATE-TOKEN: $SOURCE_TOKEN" "$protocol://$hostname/api/v4/user" | jq -r '.username')
  if [ -z "$USER_NAME" ]; then
    printf "\n\033[31m[ERROR] USER_NAME is empty\033[0m\n"
    exit 0
  fi

  _DATA=$(curl -fsSL -H "PRIVATE-TOKEN:$SOURCE_TOKEN" "$SOURCE_URL")
  echo "$_DATA" | jq -c '.[]' | while read -r row; do
    _full_path=$(echo "$row" | jq -r '.path_with_namespace')
    _description=$(echo "$row" | jq -r '.description')
    _git_url_to_repo=$(echo "$row" | jq -r '.http_url_to_repo')  

    _desc=$(echo "$_description" | head -n 1 | awk -F ' ' '{print $1}')
    if echo "$_desc" | grep  -v -q '^::'; then
      echo "skip: $_full_path"
      continue
    fi

    # ::org/repo
    # ::github@org/repo
    # ::gitcode.com@org/repo
    IFS='@' read -r CODE_HUB CODE_REPO <<< "${_desc//::/}"

    # default github
    if [ -z "$CODE_REPO" ]; then
      CODE_REPO="$CODE_HUB"
      CODE_HUB="github"
    fi   

    write_repo_file "$CODE_HUB" "$CODE_REPO" "$_git_url_to_repo" "true"
  done
}

get_repos_from_wiki_home() {
    if [ -z "${CI_PROJECT_PATH:-}" ] || [ -z "${CI_SERVER_HOST:-}" ]; then
      printf "\n\033[31m[ERROR] CI_PROJECT_PATH or CI_SERVER_HOST is empty\033[0m\n"
      exit 0
    fi

    SOURCE_TOKEN="${SOURCE_TOKEN:-}"
    if [ -n "$SOURCE_TOKEN" ]; then # 存在 TOKEN
      # 判断是 user:token 还是 token
      if echo "$SOURCE_TOKEN" | grep -q ':'; then
        _wiki_url="https://$SOURCE_TOKEN@$CI_SERVER_HOST/$CI_PROJECT_PATH.wiki.git"
      else
        if [ -z "${GITLAB_USER_LOGIN:-}" ]; then
          printf "\n\033[31m[ERROR] GITLAB_USER_LOGIN is empty\033[0m\n"
          exit 0
        fi
        _wiki_url="https://$GITLAB_USER_LOGIN:$SOURCE_TOKEN@$CI_SERVER_HOST/$CI_PROJECT_PATH.wiki.git"
      fi
    else # 从公开的 wiki 获取
      _wiki_url="https://$CI_SERVER_HOST/$CI_PROJECT_PATH.wiki.git"
    fi

    echo "FROM WIKI HOME: $_wiki_url"

    get_repos_from_git "$_wiki_url"
}

get_repos_from_wiki_git() {
    IFS=',' read -r _wiki_url _md_file <<< "$SOURCE_URL"

    echo "FROM WIKI GIT: $_wiki_url"

    get_repos_from_git "$_wiki_url" "${_md_file:-}"
}

get_repos_from_git() {
    WIKI_GIT_URL="${1:-}"
    REPO_FILE_PATH="${2:-home.md}"

    if [ -z "$WIKI_GIT_URL" ]; then
      printf "\n\033[31m[ERROR] WIKI_GIT_URL is empty\033[0m\n"
      exit 0
    fi

    target_wiki_dir=$(mktemp -u -t wiki-XXXXXX)
    git clone "$WIKI_GIT_URL" "$target_wiki_dir"

    markdown_path="$target_wiki_dir/$REPO_FILE_PATH"
    if [ -f "$markdown_path" ]; then
      SOURCE_URL="$markdown_path"
      if [[ "$SOURCE_URL" == *.json ]]; then # json
        get_repos_from_json
      else
        get_repos_from_markdown
      fi
    else
      printf "\n\033[31m[ERROR] WIKI: %s is not a valid Markdown file\033[0m\n" "$markdown_path"
      exit 0
    fi      
}

get_repos_from_markdown() {
    echo "FROM MARKDOWN: $SOURCE_URL"

    get_source_data "$SOURCE_URL"

    progress_markdown "$DATA"
}

get_repos_from_json() {
  echo "FROM JSON: $SOURCE_URL"

  get_source_data "$SOURCE_URL"

  # 判断是否为 DATA 是否为 JSON 格式
  if ! echo "$DATA" | jq '.' >/dev/null 2>&1; then
    printf "\n\033[31m[ERROR] DATA is not a valid JSON\033[0m\n"
    exit 0
  fi

  # 使用 jq 解析 JSON 并遍历
  echo "$DATA" | jq -c '.[]' | while read -r row; do
    CODE_HUB=$(echo "$row" | jq -r '.source_hub')
    CODE_REPO=$(echo "$row" | jq -r '.source_repo')
    TARGET_REPO=$(echo "$row" | jq -r '.target_repo')
    STATUS=$(echo "$row" | jq -r '.status')

    write_repo_file "$CODE_HUB" "$CODE_REPO" "$TARGET_REPO" "$STATUS"
  done
}

main() {
  if [ -f "./.env" ]; then
    # shellcheck source=/dev/null
    source "./.env"
  fi

  # Git User Name
  if [ -n "${GIT_USERNAME:-}" ]; then
    USER_NAME="${GIT_USERNAME:-}" 
  fi
  # From GitLab
  if [ -z "${USER_NAME:-}" ]; then
    USER_NAME="${GITLAB_USER_LOGIN:-}"  
  fi

  # Git User Token
  if [ -n "${GIT_USER_TOKEN:-}" ]; then
    USER_TOKEN="${GIT_USER_TOKEN:-}"
  fi

  # Git Server Host
  if [ -n "${GIT_SERV_HOST:-}" ]; then
    SERVER_HOST="${GIT_SERV_HOST:-}"  
  fi
  # From GitLab
  if [ -z "${SERVER_HOST:-}" ]; then
    SERVER_HOST="${CI_SERVER_HOST:-}"  
  fi

  # 源 URL
  SOURCE_URL="${SOURCE_URL:-}"    
  
  # WIKI 或 API TOKEN
  SOURCE_TOKEN=${SOURCE_TOKEN:-}

  # 判断是否为 API
  IS_API=""
  if echo "$SOURCE_URL" | grep -q "/api/"; then
    IS_API=1
    USER_TOKEN="${SOURCE_TOKEN:-}"
    USER_NAME="FROM_API"
    SERVER_HOST="FROM_API"
  fi

  # 最终源列表
  REPO_LIST="repos.list" 

  echo "USER_NAME: $USER_NAME"
  echo "USER_TOKEN: $USER_TOKEN"
  echo "SERVER_HOST: $SERVER_HOST"
  echo "SOURCE_URL: $SOURCE_URL"

  if [ -z "$USER_NAME" ] || [ -z "$USER_TOKEN" ] || [ -z "$SERVER_HOST" ]; then
    echo "[ERROR] USER_NAME, USER_TOKEN, SERVER_HOST must be set"
    exit 0
  fi

  if [ -f "$REPO_LIST" ]; then
    rm -rf "$REPO_LIST"
  fi

  if [ -n "$IS_API" ]; then
    get_repos_api
  else
    get_repos
  fi

  if [ ! -f "$REPO_LIST" ]; then 
    printf "\n\033[31m[ERROR] %s not found\033[0m\n" "$REPO_LIST"
    exit 0
  fi
  
  git config --global init.defaultBranch main
  git config --global pull.rebase false

  git config --global url."https://$USER_NAME:$USER_TOKEN@$SERVER_HOST".insteadOf "https://$SERVER_HOST"
  
  while IFS="|" read -r SOURCE_HUB SOURCE_REPO TARGET_REPO _; do
    case "$SOURCE_HUB" in
    github | gitlab | gitee | gitcode )
      SOURCE_CODE_URL="https://$SOURCE_HUB.com/$SOURCE_REPO.git"
      ;;
    *)
      SOURCE_CODE_URL="https://$SOURCE_HUB/$SOURCE_REPO.git"
      ;;
    esac

    # 镜像目标仓库
    if is_url "$TARGET_REPO"; then
      TARGET_CODE_URL="$TARGET_REPO"
    else
      TARGET_CODE_URL="https://$SERVER_HOST/$TARGET_REPO.git"
    fi

    echo "SOURCE_CODE_URL: $SOURCE_CODE_URL"
    echo "TARGET_CODE_URL: $TARGET_CODE_URL"

    target_dir=$(mktemp -u -t clone-XXXXXX)
    git clone "$SOURCE_CODE_URL" "$target_dir"

    pushd "$target_dir" > /dev/null || {
      printf "\n\033[31mnot found source code: %s\033[0m\n" "$SOURCE_CODE_URL"
      exit
    }

      # 获取所有远程分支
      git fetch --all

      # 更新所有本地分支，确保与远程分支同步
      while IFS= read -r branch; do
        # 获取分支名（去掉远程仓库名）
        local_branch=${branch#origin/}
        
        # 创建并切换到本地分支
        git checkout -b "$local_branch" "$branch" --no-track
        
        # 拉取最新代码
        git pull origin "$local_branch"
      done < <(git branch -r | grep -v '\->' | tr -d ' ')

      git remote add target "$TARGET_CODE_URL"
      git push target --all --force
      git push target --tags --force
    popd > /dev/null

  done < "$REPO_LIST"
}

main "$@" || exit 1

jetsung

同步本组织项目

从源站拉取代码，并更新至本组织。

配置

设置 CI 环境变量 USER_TOKEN，值为当前用户的 token。
（可选）设置定时任务。在 作业 -> 流水线计划 中创建定时任务。

本组织下的各项目信息内容设置为以下格式开头

 ::org/repo
 ::github@org/repo
 ::gitcode.com@org/repo

jetsung

Drone

.drone.yml

kind: pipeline
type: docker
name: default

clone:
  disable: true

steps:
  - name: sync  

    image: alpine:latest
    environment:
      GIT_USERNAME:
        from_secret: GIT_USERNAME
      GIT_USER_TOKEN:
        from_secret: GIT_USER_TOKEN
      GIT_SERV_HOST:
        from_secret: GIT_SERV_HOST
      SOURCE_URL:
        from_secret: SOURCE_URL
      SOURCE_TOKEN:
        from_secret: SOURCE_TOKEN      

      DRONE_REPO_TOKEN:
        from_secret: DRONE_REPO_TOKEN 
    commands: 
    - |
      apk update
      apk add curl git jq bash
    - hostname=$(echo "$CI_REPO_REMOTE" | sed -E 's#^https?://([^/]+).*#\1#')
    - git config --global init.defaultBranch main
    - git config --global pull.rebase false    

    - git config --global url."https://$DRONE_REPO_OWNER:$DRONE_REPO_TOKEN@$hostname".insteadOf "https://$hostname"
    - git init
    - git remote add origin "$CI_REPO_REMOTE"
    - git pull origin "$DRONE_SOURCE_BRANCH"
    - bash ./.codesync.sh
```