GitLab CI 代码同步（镜像同步）到仓库

jetsung · 2024-09-12T12:56:53+00:00

GitLab CI 代码同步（镜像同步）到仓库设置 USER_TOKEN：CI/CD 设置 -> 变量 ->设置“键”和“值” 仓库中添加 .gitlab-ci.yaml 文件。通用项目同步 Git 仓库到本组织名下（从 wiki）同步 Git 仓库到本组织名下（从 Git 仓库的描述）...

jetsung

GitLab CI 代码同步（镜像同步）到仓库

设置 USER_TOKEN：CI/CD 设置 -> 变量 ->设置“键”和“值”
仓库中添加 .gitlab-ci.yaml 文件。

通用项目

特定项目

jetsung

WordPress 代码同步（官网代码）

sync.sh

#!/usr/bin/env sh

set -eu

GIT_EMAIL="$GITLAB_USER_EMAIL"
GIT_USERNAME="$GITLAB_USER_NAME"
HTTPS_REPO_URL="https://$CI_SERVER_HOST/$CI_PROJECT_PATH.git"
SOURCE_URL="https://wordpress.org"
SAVE_PATH="source.zip"
SOURCE_HTML=$(curl -fsL "$SOURCE_URL/download/")

# https://wordpress.org/latest.zip
DOWN_URL="$SOURCE_URL/latest.zip"
FLAG=$(echo "$SOURCE_HTML" | sed -n '/softwareVersion/p' | cut -d '"' -f 4)

echo "tag: $FLAG"

git fetch origin --tags
if git tag | grep "$FLAG"; then
  echo
  echo "$ the tag $FLAG already exists"
  exit 0
fi

echo "downloading $DOWN_URL to $SAVE_PATH"
curl -o "$SAVE_PATH" -fsSL "$DOWN_URL"
if [ ! -f "$SAVE_PATH" ]; then
  echo "not found $SAVE_PATH"
  exit 1
fi

SOURCE_PATH="$FLAG/wordpress"
BRANCH_TEMP="temp"

echo "source path: $SOURCE_PATH"
unzip -o -q $SAVE_PATH -d "$FLAG"
if [ ! -d "$SOURCE_PATH" ]; then
  echo "not found $SOURCE_PATH"
  exit 1
fi

cd "$SOURCE_PATH" || exit 1
# code init
git init
git remote add origin "$HTTPS_REPO_URL"
git config --global user.email "$GIT_EMAIL"
git config --global user.name "$GIT_USERNAME"
git config --list

git config --global url."https://$GITLAB_USER_LOGIN:$USER_TOKEN@$CI_SERVER_HOST".insteadOf https://"$CI_SERVER_HOST"

# temp branch
git checkout -b "$BRANCH_TEMP"
git add .
git commit -am "release $FLAG"
git fetch

# merge to develop and push it
if git branch -a | grep develop; then
  git fetch origin develop
  git checkout develop
  git checkout "$BRANCH_TEMP" -- .
  git add .
  git commit -am "merge $FLAG"
else
  git branch -m develop
fi

# push develop
git push origin develop

# push tag
git tag "$FLAG"
git push origin "$FLAG"

gitlab-ci.yaml

default:
  image: alpine:latest

stages:
  - build

build-sync-job:
  stage: build
  before_script: |
    apk update
    apk add curl git
  script: |
    GIT_EMAIL="$GITLAB_USER_EMAIL"
    GIT_USERNAME="$GITLAB_USER_NAME"
    HTTPS_REPO_URL="https://$CI_SERVER_HOST/$CI_PROJECT_PATH.git"
    SOURCE_URL="https://wordpress.org"
    SAVE_PATH="source.zip"
    SOURCE_HTML=$(curl -fsL "$SOURCE_URL/download/")

    # https://wordpress.org/latest.zip
    DOWN_URL="$SOURCE_URL/latest.zip"
    FLAG=$(echo "$SOURCE_HTML" | sed -n '/softwareVersion/p' | cut -d '"' -f 4)

    echo "tag: $FLAG"

    git fetch origin --tags
    if git tag | grep "$FLAG"; then
      echo
      echo "$ the tag $FLAG already exists"
      exit 0
    fi

    echo "downloading $DOWN_URL to $SAVE_PATH"
    curl -o "$SAVE_PATH" -fsSL "$DOWN_URL"
    if [ ! -f "$SAVE_PATH" ]; then
      echo "not found $SAVE_PATH"
      exit 1
    fi

    SOURCE_PATH="$FLAG/wordpress"
    BRANCH_TEMP="temp"

    echo "source path: $SOURCE_PATH"
    unzip -o -q $SAVE_PATH -d "$FLAG"
    if [ ! -d "$SOURCE_PATH" ]; then
      echo "not found $SOURCE_PATH"
      exit 1
    fi

    cd "$SOURCE_PATH" || exit 1
    # code init
    git init
    git remote add origin "$HTTPS_REPO_URL"
    git config --global user.email "$GIT_EMAIL"
    git config --global user.name "$GIT_USERNAME"
    git config --list

    git config --global url."https://$GITLAB_USER_LOGIN:$USER_TOKEN@$CI_SERVER_HOST".insteadOf https://"$CI_SERVER_HOST"

    # temp branch
    git checkout -b "$BRANCH_TEMP"
    git add .
    git commit -am "release $FLAG"
    git fetch

    # merge to develop and push it
    if git branch -a | grep develop; then
      git fetch origin develop
      git checkout develop
      git checkout "$BRANCH_TEMP" -- .
      git add .
      git commit -am "merge $FLAG"
    else
      git branch -m develop
    fi

    # push develop
    git push origin develop

    # push tag
    git tag "$FLAG"
    git push origin "$FLAG"

jetsung

DouPHP 代码同步（官网代码）

https://www.douphp.com/

sync.sh

#!/usr/bin/env sh

set -eu

GIT_EMAIL="$GITLAB_USER_EMAIL"
GIT_USERNAME="$GITLAB_USER_NAME"
HTTPS_REPO_URL="https://$CI_SERVER_HOST/$CI_PROJECT_PATH.git"
SOURCE_URL="https://www.douphp.com/"
SAVE_PATH="source.zip"

SOURCE_HTML=$(curl -fsSL "$SOURCE_URL")

# https://down.douphp.com/DouPHP_1.7_Release_20231214.zip
DOWN_URL=$(echo "$SOURCE_HTML" | sed -n '/fa-download/p' | grep -o 'http[s]://down.[^"]*')
FLAG=$(echo "${DOWN_URL##*/}" | awk -F'_' '{print $2"_"$4}' | sed 's#.zip##g')

echo "tag: $FLAG"

git fetch origin --tags
if git tag | grep "$FLAG"; then
  echo
  echo "$ the tag $FLAG already exists"
  exit 0
fi

echo "downloading $DOWN_URL to $SAVE_PATH"
curl -o "$SAVE_PATH" -fsSL "$DOWN_URL"
if [ ! -f "$SAVE_PATH" ]; then
  echo "not found $SAVE_PATH"
  exit 1
fi

SOURCE_PATH="$FLAG/upload"
BRANCH_TEMP="temp"

echo "source path: $SOURCE_PATH"
unzip -o -q $SAVE_PATH -d "$FLAG"
if [ ! -d "$SOURCE_PATH" ]; then
  echo "not found $SOURCE_PATH"
  exit 1
fi

cd "$SOURCE_PATH" || exit 1
# code init
git init
git remote add origin "$HTTPS_REPO_URL"
git config --global user.email "$GIT_EMAIL"
git config --global user.name "$GIT_USERNAME"
git config --list

git config --global url."https://$GITLAB_USER_LOGIN:$USER_TOKEN@$CI_SERVER_HOST".insteadOf https://"$CI_SERVER_HOST"

# temp branch
git checkout -b "$BRANCH_TEMP"
git add .
git commit -am "release $FLAG"
git fetch

# merge to develop and push it
if git branch -a | grep develop; then
  git fetch origin develop
  git checkout develop
  git checkout "$BRANCH_TEMP" -- .
  git add .
  git commit -am "merge $FLAG"
else
  git branch -m develop
fi

# push develop
git push origin develop

# push tag
git tag "$FLAG"
git push origin "$FLAG"

.gitlab-ci.yaml

default:
  image: alpine:latest

stages:
  - build

build-sync-job:
  stage: build
  before_script: |
    apk update
    apk add curl git
  script: |
    GIT_EMAIL="$GITLAB_USER_EMAIL"
    GIT_USERNAME="$GITLAB_USER_NAME"
    HTTPS_REPO_URL="https://$CI_SERVER_HOST/$CI_PROJECT_PATH.git"
    SOURCE_URL="https://www.douphp.com/"
    SAVE_PATH="source.zip"

    SOURCE_HTML=$(curl -fsSL "$SOURCE_URL")

    # https://down.douphp.com/DouPHP_1.7_Release_20231214.zip
    DOWN_URL=$(echo "$SOURCE_HTML" | sed -n '/fa-download/p' | grep -o 'http[s]://down.[^"]*')
    FLAG=$(echo "${DOWN_URL##*/}" | awk -F'_' '{print $2"_"$4}' | sed 's#.zip##g')

    echo "tag: $FLAG"

    git fetch origin --tags
    if git tag | grep "$FLAG"; then
      echo
      echo "$ the tag $FLAG already exists"
      exit 0
    fi

    echo "downloading $DOWN_URL to $SAVE_PATH"
    curl -o "$SAVE_PATH" -fsSL "$DOWN_URL"
    if [ ! -f "$SAVE_PATH" ]; then
      echo "not found $SAVE_PATH"
      exit 1
    fi

    SOURCE_PATH="$FLAG/upload"
    BRANCH_TEMP="temp"

    echo "source path: $SOURCE_PATH"
    unzip -o -q $SAVE_PATH -d "$FLAG"
    if [ ! -d "$SOURCE_PATH" ]; then
      echo "not found $SOURCE_PATH"
      exit 1
    fi

    cd "$SOURCE_PATH" || exit 1
    # code init
    git init
    git remote add origin "$HTTPS_REPO_URL"
    git config --global user.email "$GIT_EMAIL"
    git config --global user.name "$GIT_USERNAME"
    git config --list

    git config --global url."https://$GITLAB_USER_LOGIN:$USER_TOKEN@$CI_SERVER_HOST".insteadOf https://"$CI_SERVER_HOST"

    # temp branch
    git checkout -b "$BRANCH_TEMP"
    git add .
    git commit -am "release $FLAG"
    git fetch

    # merge to develop and push it
    if git branch -a | grep develop; then
      git fetch origin develop
      git checkout develop
      git checkout "$BRANCH_TEMP" -- .
      git add .
      git commit -am "merge $FLAG"
    else
      git branch -m develop
    fi

    # push develop
    git push origin develop

    # push tag
    git tag "$FLAG"
    git push origin "$FLAG"

jetsung

同步 Git 仓库到本组织名下（从 wiki）

从源站拉取代码，并更新到指定目标仓库。

设置 CI 环境变量 USER_TOKEN，值为当前用户的 token。
（可选）设置定时任务。在 作业 -> 流水线计划 中创建定时任务。

仓库列表，位于 wikis/home，格式如下：

| 源 | 源仓库 | 目标仓库 | 状态 |

源：github, gitlab, gitee
源仓库：源仓库地址（不含网址），如 org/repo
目标仓库：目标仓库地址（不含网址），如 org/repo
状态：是否同步，true 表示同步，其它则表示不同步

sync.sh

#!/usr/bin/env bash

set -e

main() {
  USER_TOKEN="${USER_TOKEN:-}"
  USER_NAME="${GITLAB_USER_LOGIN:-}"
  TARGET_HUB="${CI_SERVER_HOST:-}"
  PROJECT_PATH="${CI_PROJECT_PATH:-}"

  if [ -z "$USER_TOKEN" ] || [ -z "$USER_NAME" ] || [ -z "$TARGET_HUB" ]; then
    echo "[ERROR] USER_TOKEN, USER_NAME, TARGET_HUB must be set"
    exit 1
  fi

  PROJECT_PATH_URL="$(echo "$PROJECT_PATH" | sed 's/\//%2F/g')"
  echo "FETCH API: https://$TARGET_HUB/api/v4/projects/$PROJECT_PATH_URL/wikis/home"

  DATA=$(curl -fsSL -H "PRIVATE-TOKEN:$USER_TOKEN" "https://$TARGET_HUB/api/v4/projects/$PROJECT_PATH_URL/wikis/home" | jq '.content')
  echo -e "$DATA" | sed 's# ##g' | while IFS="|" read -r _ ORIGIN_HUB ORIGIN_REPO TARGET_REPO STATUS; do
    CODE_HUB=$(echo "$ORIGIN_HUB" | cut -d '`' -f 2)
    CODE_REPO=$(echo "$ORIGIN_REPO" | cut -d '`' -f 2)
    TARGET_REPO=$(echo "$TARGET_REPO" | cut -d '`' -f 2)
    STATUS=${STATUS%%|*}
    CODE_HUB="${CODE_HUB:-github}"
    CODE_REPO="${CODE_REPO:-}"
    TARGET_REPO="${TARGET_REPO:-}"
    STATUS="${STATUS:-}"
    if [ -z "$CODE_HUB" ] || [ -z "$CODE_REPO" ] || [ -z "$TARGET_REPO" ]; then
      continue
    fi

    if [ "$STATUS" != "true" ]; then
      continue
    fi

    case $CODE_HUB in
    github | gitlab | gitee)
      CODE_URL="https://$CODE_HUB.com/$CODE_REPO.git"
      ;;
    *)
      echo "custom code hub: $CODE_HUB"
      CODE_URL="https://$CODE_HUB/$CODE_REPO.git"
      # continue
      ;;
    esac

    HTTPS_REPO_URL="https://$USER_NAME:$USER_TOKEN@$TARGET_HUB/$TARGET_REPO.git"
    echo "CODE_URL: $CODE_URL"
    echo "HTTPS_REPO_URL: $HTTPS_REPO_URL"

    [ -d procode ] && rm -rf procode
    git clone "$CODE_URL" procode
    cd procode || {
      echo "not found procode"
      exit 1
    }

    git remote add target "$HTTPS_REPO_URL"
    git fetch
    git push target --all --force
    git push target --tags --force
    cd ..
  done
}

main "$@" || exit 1

.gitlab-ci.yaml

default:
  image: alpine:latest

stages:
  - build

build-sync-job:
  stage: build
  before_script: |
    apk update
    apk add curl git jq
  script: |
    USER_TOKEN="${USER_TOKEN:-}"
    USER_NAME="${GITLAB_USER_LOGIN:-}"
    TARGET_HUB="${CI_SERVER_HOST:-}"
    PROJECT_PATH="${CI_PROJECT_PATH:-}"

    if [ -z "$USER_TOKEN" ] || [ -z "$USER_NAME" ] || [ -z "$TARGET_HUB" ]; then
      echo "[ERROR] USER_TOKEN, USER_NAME, TARGET_HUB must be set"
      exit 1
    fi

    PROJECT_PATH_URL="$(echo "$PROJECT_PATH" | sed 's/\//%2F/g')"
    echo "FETCH API: https://$TARGET_HUB/api/v4/projects/$PROJECT_PATH_URL/wikis/home"

    DATA=$(curl -fsSL -H "PRIVATE-TOKEN:$USER_TOKEN" "https://$TARGET_HUB/api/v4/projects/$PROJECT_PATH_URL/wikis/home" | jq '.content')
    echo -e "$DATA" | sed 's# ##g' | while IFS="|" read -r _ ORIGIN_HUB ORIGIN_REPO TARGET_REPO STATUS; do
      CODE_HUB=$(echo "$ORIGIN_HUB" | cut -d '`' -f 2)
      CODE_REPO=$(echo "$ORIGIN_REPO" | cut -d '`' -f 2)
      TARGET_REPO=$(echo "$TARGET_REPO" | cut -d '`' -f 2)
      STATUS=${STATUS%%|*}
      CODE_HUB="${CODE_HUB:-github}"
      CODE_REPO="${CODE_REPO:-}"
      TARGET_REPO="${TARGET_REPO:-}"
      STATUS="${STATUS:-}"
      if [ -z "$CODE_HUB" ] || [ -z "$CODE_REPO" ] || [ -z "$TARGET_REPO" ]; then
        continue
      fi

      if [ "$STATUS" != "true" ]; then
        continue
      fi

      case $CODE_HUB in
      github | gitlab | gitee)
        CODE_URL="https://$CODE_HUB.com/$CODE_REPO.git"
        ;;
      *)
        echo "custom code hub: $CODE_HUB"
        CODE_URL="https://$CODE_HUB/$CODE_REPO.git"
        # continue
        ;;
      esac

      HTTPS_REPO_URL="https://$USER_NAME:$USER_TOKEN@$TARGET_HUB/$TARGET_REPO.git"
      echo "CODE_URL: $CODE_URL"
      echo "HTTPS_REPO_URL: $HTTPS_REPO_URL"

      [ -d procode ] && rm -rf procode
      git clone "$CODE_URL" procode
      cd procode || {
        echo "not found procode"
        exit 1
      }

      git remote add target "$HTTPS_REPO_URL"
      git fetch
      git push target --all --force
      git push target --tags --force
      cd ..
    done

jetsung

同步 Git 仓库到本组织名下（从 Git 仓库的描述）

同步本组织项目

从源站拉取代码，并更新至本组织。

设置 CI 环境变量 USER_TOKEN，值为当前用户的 token。
（可选）设置定时任务。在 作业 -> 流水线计划 中创建定时任务。

在仓库的描述中设置

::org/repo

sync.sh

#!/usr/bin/env sh

set -e

main() {
  GIT_URL="https://$CI_SERVER_HOST"
  ORG_NAME="$CI_PROJECT_NAMESPACE"
  curl -o projects.json -H "PRIVATE-TOKEN:$USER_TOKEN" "$GIT_URL/api/v4/groups/$ORG_NAME/projects"
  JSON_DATA=$(cat projects.json)

  git config --global url."https://$GITLAB_USER_LOGIN:$USER_TOKEN@$CI_SERVER_HOST".insteadOf "$GIT_URL"

  for entry in $(echo "$JSON_DATA" | jq -r '.[] | @base64'); do
    decoded_entry=$(echo "$entry" | base64 -d)
    path=$(echo "$decoded_entry" | jq -r '.path')
    description=$(echo "$decoded_entry" | jq -r '.description')
    HTTPS_REPO_URL=$(echo "$decoded_entry" | jq -r '.http_url_to_repo')

    case $description in
    "::"*)
      echo "-> [$path] sync ..."

      # origin repo -> ::gitlab@org/repo THIS IS SYNC PROJECT
      SYNC_VARS="$(echo "$description" | head -n 1 | awk -F ' ' '{print $1}' | sed 's#::##')"
      ORIGIN_HUB=$(echo "$SYNC_VARS" | awk -F@ '{print $1}')
      ORIGIN_REPO=$(echo "$SYNC_VARS" | awk -F@ '{print $2}')

      # default github
      if [ -z "$ORIGIN_REPO" ]; then
        ORIGIN_REPO="$ORIGIN_HUB"
        ORIGIN_HUB="github"
      fi

      case $ORIGIN_HUB in
      github | gitlab | gitee)
        CODE_URL="https://$ORIGIN_HUB.com/$ORIGIN_REPO.git"
        ;;
      *)
        # echo "unknown code hub: $ORIGIN_HUB"
        # return
        CODE_URL="https://$ORIGIN_HUB/$ORIGIN_REPO.git"
        ;;
      esac

      echo "HTTPS_REPO_URL: $HTTPS_REPO_URL"

      [ -d procode ] && rm -rf procode
      git clone "$CODE_URL" procode
      cd procode
      git remote add target "$HTTPS_REPO_URL"
      git fetch
      git push target --all --force
      git push target --tags --force
      cd ..
      ;;
    # *) echo "path: $path skip sync." ;;
    esac
  done
}

main "$@" || exit 1

.gitlab-ci.yaml

default:
  image: alpine:latest

stages:
  - sync

sync-job:
  stage: sync
  before_script: |
    apk update
    apk add curl git jq
  script: |
    GIT_URL="https://$CI_SERVER_HOST"
    ORG_NAME="$CI_PROJECT_NAMESPACE"
    curl -o projects.json -H "PRIVATE-TOKEN:$USER_TOKEN" "$GIT_URL/api/v4/groups/$ORG_NAME/projects"
    JSON_DATA=$(cat projects.json)

    git config --global url."https://$GITLAB_USER_LOGIN:$USER_TOKEN@$CI_SERVER_HOST".insteadOf "$GIT_URL"

    for entry in $(echo "$JSON_DATA" | jq -r '.[] | @base64'); do
      decoded_entry=$(echo "$entry" | base64 -d)
      path=$(echo "$decoded_entry" | jq -r '.path')
      description=$(echo "$decoded_entry" | jq -r '.description')
      HTTPS_REPO_URL=$(echo "$decoded_entry" | jq -r '.http_url_to_repo')

      case $description in
      "::"*)
        echo "-> [$path] sync ..."

        # origin repo -> ::gitlab@org/repo THIS IS SYNC PROJECT
        SYNC_VARS="$(echo "$description" | head -n 1 | awk -F ' ' '{print $1}' | sed 's#::##')"
        ORIGIN_HUB=$(echo "$SYNC_VARS" | awk -F@ '{print $1}')
        ORIGIN_REPO=$(echo "$SYNC_VARS" | awk -F@ '{print $2}')

        # default github
        if [ -z "$ORIGIN_REPO" ]; then
          ORIGIN_REPO="$ORIGIN_HUB"
          ORIGIN_HUB="github"
        fi

        case $ORIGIN_HUB in
        github | gitlab | gitee)
          CODE_URL="https://$ORIGIN_HUB.com/$ORIGIN_REPO.git"
          ;;
        *)
          # echo "unknown code hub: $ORIGIN_HUB"
          # return
          CODE_URL="https://$ORIGIN_HUB/$ORIGIN_REPO.git"
          ;;
        esac

        echo "HTTPS_REPO_URL: $HTTPS_REPO_URL"

        [ -d procode ] && rm -rf procode
        git clone "$CODE_URL" procode
        cd procode
        git remote add target "$HTTPS_REPO_URL"
        git fetch
        git push target --all --force
        git push target --tags --force
        cd ..
        ;;
      # *) echo "path: $path skip sync." ;;
      esac
    done