git-push-all.sh 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309
  1. #!/usr/bin/env bash
  2. SCRIPT_NAME=$(basename "$0")
  3. function usage()
  4. {
  5. if [ "$TOR_PUSH_SAME" ]; then
  6. CURRENT_PUSH_SAME="push"
  7. else
  8. CURRENT_PUSH_SAME="skip"
  9. fi
  10. echo "$SCRIPT_NAME [-h] [-r <remote-name> [-t <test-branch-prefix>]] [-s]"
  11. # The next line looks misaligned, but it lines up in the output
  12. echo " [-- [-n] [--no-atomic] <git push options>]"
  13. echo
  14. echo " arguments:"
  15. echo " -h: show this help text"
  16. echo " -n: dry run mode"
  17. echo " (default: run commands)"
  18. echo " -r: push to remote-name, rather than the default upstream remote."
  19. echo " (default: $DEFAULT_UPSTREAM_REMOTE, current: $UPSTREAM_REMOTE)"
  20. echo " -t: test branch mode: push test branches to remote-name. Pushes"
  21. echo " branches prefix_035, prefix_040, ... , prefix_main."
  22. echo " (default: push maint-*, release-*, and main)"
  23. echo " -s: push branches whose tips match upstream maint, release, or"
  24. echo " main branches. The default is to skip these branches,"
  25. echo " because they do not contain any new code. Use -s to test for"
  26. echo " CI environment failures, using code that previously passed CI."
  27. echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)"
  28. echo " --: pass further arguments to git push."
  29. echo " All unrecognised arguments are passed to git push, but complex"
  30. echo " arguments before -- may be mangled by getopt."
  31. echo " (default: git push --atomic, current: $GIT_PUSH)"
  32. echo
  33. echo " env vars:"
  34. echo " optional:"
  35. echo " TOR_GIT_PUSH_PATH: change to this directory before pushing."
  36. echo " (default: if \$TOR_FULL_GIT_PATH is set,"
  37. echo " use \$TOR_FULL_GIT_PATH/\$TOR_MASTER;"
  38. echo " Otherwise, use the current directory for pushes;"
  39. echo " current: $TOR_GIT_PUSH_PATH)"
  40. echo " TOR_FULL_GIT_PATH: where the git repository directories reside."
  41. echo " We recommend using \$HOME/git/."
  42. echo " (default: use the current directory for pushes;"
  43. echo " current: $TOR_FULL_GIT_PATH)"
  44. echo " TOR_MASTER: the name of the directory containing the tor.git clone"
  45. echo " The primary tor git directory is \$GIT_PATH/\$TOR_MASTER"
  46. echo " (default: tor; current: $TOR_MASTER_NAME)"
  47. echo
  48. echo " TOR_UPSTREAM_REMOTE_NAME: the default upstream remote."
  49. echo " Overridden by -r."
  50. echo " (default: upstream; current: $UPSTREAM_REMOTE)"
  51. echo " TOR_GIT_PUSH: the git push command and default arguments."
  52. echo " Overridden by <git push options> after --."
  53. echo " (default: git push --atomic; current: $GIT_PUSH)"
  54. echo " TOR_PUSH_SAME: push branches whose tips match upstream maint,"
  55. echo " release, or main branches. Inverted by -s."
  56. echo " (default: skip; current: $CURRENT_PUSH_SAME matching branches)"
  57. echo " TOR_PUSH_DELAY: pushes the main and maint branches separately,"
  58. echo " so that CI runs in a sensible order."
  59. echo " (default: push all branches immediately; current: $PUSH_DELAY)"
  60. echo " we recommend that you set these env vars in your ~/.profile"
  61. }
  62. set -e
  63. #################
  64. # Configuration #
  65. #################
  66. # Don't change this configuration - set the env vars in your .profile
  67. #
  68. # The primary tor git repository directory from which all the worktree have
  69. # been created.
  70. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"}
  71. # Which directory do we push from?
  72. if [ "$TOR_FULL_GIT_PATH" ]; then
  73. TOR_GIT_PUSH_PATH=${TOR_GIT_PUSH_PATH:-"$TOR_FULL_GIT_PATH/$TOR_MASTER_NAME"}
  74. fi
  75. # git push command and default arguments
  76. GIT_PUSH=${TOR_GIT_PUSH:-"git push --atomic"}
  77. # The upstream remote which gitlab.torproject.org/tpo/core/tor.git points to.
  78. DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"}
  79. # Push to a different upstream remote using -r <remote-name>
  80. UPSTREAM_REMOTE=${DEFAULT_UPSTREAM_REMOTE}
  81. # Add a delay between pushes, so CI runs on the most important branches first
  82. PUSH_DELAY=${TOR_PUSH_DELAY:-0}
  83. # Push (1) or skip (0) test branches that are the same as an upstream
  84. # maint/main branch. Push if you are testing that the CI environment still
  85. # works on old code, skip if you are testing new code in the branch.
  86. # Default: skip unchanged branches.
  87. # Inverted by the -s option.
  88. PUSH_SAME=${TOR_PUSH_SAME:-0}
  89. #######################
  90. # Argument processing #
  91. #######################
  92. # Controlled by the -t <test-branch-prefix> option. The test branch prefix
  93. # option makes git-merge-forward.sh create new test branches:
  94. # <tbp>_035, <tbp>_040, ... , <tbp>_main, and merge each branch forward into
  95. # the next one.
  96. TEST_BRANCH_PREFIX=
  97. while getopts ":hr:st:" opt; do
  98. case "$opt" in
  99. h) usage
  100. exit 0
  101. ;;
  102. r) UPSTREAM_REMOTE="$OPTARG"
  103. echo " *** PUSHING TO REMOTE: ${UPSTREAM_REMOTE} ***"
  104. shift
  105. shift
  106. OPTIND=$((OPTIND - 2))
  107. ;;
  108. s) PUSH_SAME=$((! PUSH_SAME))
  109. if [ "$PUSH_SAME" -eq 0 ]; then
  110. echo " *** SKIPPING UNCHANGED TEST BRANCHES ***"
  111. else
  112. echo " *** PUSHING UNCHANGED TEST BRANCHES ***"
  113. fi
  114. shift
  115. OPTIND=$((OPTIND - 1))
  116. ;;
  117. t) TEST_BRANCH_PREFIX="$OPTARG"
  118. echo " *** PUSHING TEST BRANCHES: ${TEST_BRANCH_PREFIX}_nnn ***"
  119. shift
  120. shift
  121. OPTIND=$((OPTIND - 2))
  122. ;;
  123. *)
  124. # Make git push handle the option
  125. # This might mangle options with spaces, use -- for complex options
  126. GIT_PUSH="$GIT_PUSH $1"
  127. shift
  128. OPTIND=$((OPTIND - 1))
  129. ;;
  130. esac
  131. done
  132. # getopts doesn't allow "-" as an option character,
  133. # so we have to handle -- manually
  134. if [ "$1" = "--" ]; then
  135. shift
  136. fi
  137. if [ "$TEST_BRANCH_PREFIX" ]; then
  138. if [ "$UPSTREAM_REMOTE" = "$DEFAULT_UPSTREAM_REMOTE" ]; then
  139. echo "Pushing test branches ${TEST_BRANCH_PREFIX}_nnn to " \
  140. "the default remote $DEFAULT_UPSTREAM_REMOTE is not allowed."
  141. echo
  142. usage
  143. exit 1
  144. fi
  145. fi
  146. if [ "$TOR_GIT_PUSH_PATH" ]; then
  147. echo "Changing to $TOR_GIT_PUSH_PATH before pushing"
  148. cd "$TOR_GIT_PUSH_PATH"
  149. else
  150. echo "Pushing from the current directory"
  151. fi
  152. echo "Calling $GIT_PUSH" "$@" "<branches>"
  153. ################################
  154. # Git upstream remote branches #
  155. ################################
  156. set -e
  157. DEFAULT_UPSTREAM_BRANCHES=
  158. if [ "$DEFAULT_UPSTREAM_REMOTE" != "$UPSTREAM_REMOTE" ]; then
  159. for br in $(git-list-tor-branches.sh -l); do
  160. DEFAULT_UPSTREAM_BRANCHES="${DEFAULT_UPSTREAM_BRANCHES} ${DEFAULT_UPSTREAM_REMOTE}/${br}"
  161. done
  162. fi
  163. UPSTREAM_BRANCHES=
  164. for br in $(git-list-tor-branches.sh -l); do
  165. UPSTREAM_BRANCHES="${UPSTREAM_BRANCHES} ${UPSTREAM_REMOTE}/${br}"
  166. done
  167. ########################
  168. # Git branches to push #
  169. ########################
  170. if [ -z "$TEST_BRANCH_PREFIX" ]; then
  171. # maint/release push mode: push all branches.
  172. #
  173. # List of branches to push. Ordering is not important.
  174. PUSH_BRANCHES="$(git-list-tor-branches.sh -l)"
  175. else
  176. # Test branch push mode: push test branches, based on each maint branch.
  177. #
  178. # List of branches to push. Ordering is not important.
  179. PUSH_BRANCHES=""
  180. for suffix in $(git-list-tor-branches.sh -s -R); do
  181. PUSH_BRANCHES="${PUSH_BRANCHES} ${TEST_BRANCH_PREFIX}${suffix}"
  182. done
  183. fi
  184. set +e
  185. ###############
  186. # Entry point #
  187. ###############
  188. if [ "$TEST_BRANCH_PREFIX" ]; then
  189. # Skip the test branches that are the same as the default or current
  190. # upstream branches (they have already been tested)
  191. UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES $DEFAULT_UPSTREAM_BRANCHES"
  192. else
  193. # Skip the local maint-*, release-*, main branches that are the same as the
  194. # current upstream branches, but ignore the default upstream
  195. # (we want to update a non-default remote, even if it matches the default)
  196. UPSTREAM_SKIP_SAME_AS="$UPSTREAM_BRANCHES"
  197. fi
  198. # Skip branches that match the relevant upstream(s)
  199. if [ "$PUSH_SAME" -eq 0 ]; then
  200. NEW_PUSH_BRANCHES=
  201. for b in $PUSH_BRANCHES; do
  202. PUSH_COMMIT=$(git rev-parse "$b")
  203. SKIP_UPSTREAM=
  204. for u in $UPSTREAM_SKIP_SAME_AS; do
  205. # Skip the branch check on error
  206. UPSTREAM_COMMIT=$(git rev-parse "$u" 2>/dev/null) || continue
  207. if [ "$PUSH_COMMIT" = "$UPSTREAM_COMMIT" ]; then
  208. SKIP_UPSTREAM="$u"
  209. fi
  210. done
  211. if [ "$SKIP_UPSTREAM" ]; then
  212. printf "Skipping unchanged: %s matching remote: %s\\n" \
  213. "$b" "$SKIP_UPSTREAM"
  214. else
  215. if [ "$NEW_PUSH_BRANCHES" ]; then
  216. NEW_PUSH_BRANCHES="${NEW_PUSH_BRANCHES} ${b}"
  217. else
  218. NEW_PUSH_BRANCHES="${b}"
  219. fi
  220. fi
  221. done
  222. PUSH_BRANCHES=${NEW_PUSH_BRANCHES}
  223. fi
  224. if [ ! "$PUSH_BRANCHES" ]; then
  225. echo "No branches to push!"
  226. # We expect the rest of the script to run without errors, even if there
  227. # are no branches
  228. fi
  229. if [ "$PUSH_DELAY" -le 0 ]; then
  230. echo "Pushing $PUSH_BRANCHES"
  231. # We know that there are no spaces in any branch within $PUSH_BRANCHES, so
  232. # it is safe to use it unquoted. (This also applies to the other shellcheck
  233. # exceptions below.)
  234. #
  235. # Push all the branches at the same time
  236. # shellcheck disable=SC2086
  237. $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $PUSH_BRANCHES
  238. else
  239. # Push the branches in optimal CI order, with a delay between each push
  240. PUSH_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | sort -V)
  241. MASTER_BRANCH=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep main$) \
  242. || true # Skipped main branch
  243. if [ -z "$TEST_BRANCH_PREFIX" ]; then
  244. MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep maint) \
  245. || true # Skipped all maint branches
  246. RELEASE_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep release | \
  247. tr "\\n" " ") || true # Skipped all release branches
  248. else
  249. # Actually test branches based on maint branches
  250. MAINT_BRANCHES=$(echo "$PUSH_BRANCHES" | tr " " "\\n" | grep -v main$) \
  251. || true # Skipped all maint test branches
  252. # No release branches
  253. RELEASE_BRANCHES=
  254. fi
  255. if [ "$MASTER_BRANCH" ] || [ "$MAINT_BRANCHES" ] \
  256. || [ "$RELEASE_BRANCHES" ]; then
  257. printf "Pushing with %ss delays, so CI runs in this order:\\n" \
  258. "$PUSH_DELAY"
  259. if [ "$MASTER_BRANCH" ]; then
  260. printf "%s\\n" "$MASTER_BRANCH"
  261. fi
  262. if [ "$MAINT_BRANCHES" ]; then
  263. printf "%s\\n" "$MAINT_BRANCHES"
  264. fi
  265. if [ "$RELEASE_BRANCHES" ]; then
  266. printf "%s\\n" "$RELEASE_BRANCHES"
  267. fi
  268. fi
  269. # shellcheck disable=SC2086
  270. for b in $MASTER_BRANCH $MAINT_BRANCHES; do
  271. $GIT_PUSH "$@" "$UPSTREAM_REMOTE" "$b"
  272. # If we are pushing more than one branch, delay. In the unlikely scenario
  273. # where we are pushing maint branches without the main branch, or maint
  274. # without release, there may be an extra delay
  275. if [ "$MAINT_BRANCHES" ] || [ "$RELEASE_BRANCHES" ]; then
  276. sleep "$PUSH_DELAY"
  277. fi
  278. done
  279. if [ "$RELEASE_BRANCHES" ]; then
  280. # shellcheck disable=SC2086
  281. $GIT_PUSH "$@" "$UPSTREAM_REMOTE" $RELEASE_BRANCHES
  282. fi
  283. fi