git-setup-dirs.sh 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539
  1. #!/usr/bin/env bash
  2. SCRIPT_NAME=$(basename "$0")
  3. function usage()
  4. {
  5. echo "$SCRIPT_NAME [-h] [-n] [-u]"
  6. echo
  7. echo " arguments:"
  8. echo " -h: show this help text"
  9. echo " -n: dry run mode"
  10. echo " (default: run commands)"
  11. echo " -u: if a directory or worktree already exists, use it"
  12. echo " (default: fail and exit on existing directories)"
  13. echo
  14. echo " env vars:"
  15. echo " required:"
  16. echo " TOR_FULL_GIT_PATH: where the git repository directories reside."
  17. echo " You must set this env var, we recommend \$HOME/git/"
  18. echo " (default: fail if this env var is not set;"
  19. echo " current: $GIT_PATH)"
  20. echo
  21. echo " optional:"
  22. echo " TOR_MASTER: the name of the directory containing the tor.git clone"
  23. echo " The tor master git directory is \$GIT_PATH/\$TOR_MASTER"
  24. echo " (default: tor; current: $TOR_MASTER_NAME)"
  25. echo " TOR_WKT_NAME: the name of the directory containing the tor"
  26. echo " worktrees. The tor worktrees are:"
  27. echo " \$GIT_PATH/\$TOR_WKT_NAME/{maint-*,release-*}"
  28. echo " (default: tor-wkt; current: $TOR_WKT_NAME)"
  29. echo " TOR_GIT_ORIGIN_PULL: the origin remote pull URL."
  30. echo " (current: $GIT_ORIGIN_PULL)"
  31. echo " TOR_GIT_ORIGIN_PUSH: the origin remote push URL"
  32. echo " (current: $GIT_ORIGIN_PUSH)"
  33. echo " TOR_UPSTREAM_REMOTE_NAME: the default upstream remote."
  34. echo " If \$TOR_UPSTREAM_REMOTE_NAME is not 'origin', we have a"
  35. echo " separate upstream remote, and we don't push to origin."
  36. echo " (default: $DEFAULT_UPSTREAM_REMOTE)"
  37. echo " TOR_GITHUB_PULL: the tor-github remote pull URL"
  38. echo " (current: $GITHUB_PULL)"
  39. echo " TOR_GITHUB_PUSH: the tor-github remote push URL"
  40. echo " (current: $GITHUB_PUSH)"
  41. echo " TOR_GITLAB_PULL: the tor-gitlab remote pull URL"
  42. echo " (current: $GITLAB_PULL)"
  43. echo " TOR_GITLAB_PUSH: the tor-gitlab remote push URL"
  44. echo " (current: $GITLAB_PUSH)"
  45. echo " TOR_EXTRA_CLONE_ARGS: extra arguments to git clone"
  46. echo " (current: $TOR_EXTRA_CLONE_ARGS)"
  47. echo " TOR_EXTRA_REMOTE_NAME: the name of an extra remote"
  48. echo " This remote is not pulled by this script or git-pull-all.sh."
  49. echo " This remote is not pushed by git-push-all.sh."
  50. echo " (current: $TOR_EXTRA_REMOTE_NAME)"
  51. echo " TOR_EXTRA_REMOTE_PULL: the extra remote pull URL."
  52. echo " (current: $TOR_EXTRA_REMOTE_PULL)"
  53. echo " TOR_EXTRA_REMOTE_PUSH: the extra remote push URL"
  54. echo " (current: $TOR_EXTRA_REMOTE_PUSH)"
  55. echo " we recommend that you set these env vars in your ~/.profile"
  56. }
  57. #################
  58. # Configuration #
  59. #################
  60. # Don't change this configuration - set the env vars in your .profile
  61. # Where are all those git repositories?
  62. GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"}
  63. # The tor master git repository directory from which all the worktree have
  64. # been created.
  65. TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"}
  66. # The worktrees location (directory).
  67. TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"}
  68. # Origin repositories
  69. GIT_ORIGIN_PULL=${TOR_GIT_ORIGIN_PULL:-"https://git.torproject.org/tor.git"}
  70. GIT_ORIGIN_PUSH=${TOR_GIT_ORIGIN_PUSH:-"git@git-rw.torproject.org:tor.git"}
  71. # The upstream remote which git.torproject.org/tor.git points to.
  72. DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"}
  73. # Copy the URLs from origin
  74. GIT_UPSTREAM_PULL="$GIT_ORIGIN_PULL"
  75. GIT_UPSTREAM_PUSH="$GIT_ORIGIN_PUSH"
  76. # And avoid pushing to origin if we have an upstream
  77. if [ "$DEFAULT_UPSTREAM_REMOTE" != "origin" ]; then
  78. GIT_ORIGIN_PUSH="No pushes to origin, if there is an upstream"
  79. fi
  80. # GitHub repositories
  81. GITHUB_PULL=${TOR_GITHUB_PULL:-"https://github.com/torproject/tor.git"}
  82. GITHUB_PUSH=${TOR_GITHUB_PUSH:-"No_Pushing_To_GitHub"}
  83. # GitLab repositories
  84. GITLAB_PULL=${TOR_GITLAB_PULL:-"https://gitlab.torproject.org/tpo/core/tor.git"}
  85. GITLAB_PUSH=${TOR_GITLAB_PUSH:-"No_Pushing_To_GitLab"}
  86. ##########################
  87. # Git branches to manage #
  88. ##########################
  89. # The branches and worktrees need to be modified when there is a new branch,
  90. # and when an old branch is no longer supported.
  91. set -e
  92. eval "$(git-list-tor-branches.sh -b)"
  93. set +e
  94. # The master branch path has to be the main repository thus contains the
  95. # origin that will be used to fetch the updates. All the worktrees are created
  96. # from that repository.
  97. ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME"
  98. #######################
  99. # Argument processing #
  100. #######################
  101. # Controlled by the -n option. The dry run option will just output the command
  102. # that would have been executed for each worktree.
  103. DRY_RUN=0
  104. # Controlled by the -s option. The use existing option checks for existing
  105. # directories, and re-uses them, rather than creating a new directory.
  106. USE_EXISTING=0
  107. USE_EXISTING_HINT="Use existing: '$SCRIPT_NAME -u'."
  108. while getopts "hnu" opt; do
  109. case "$opt" in
  110. h) usage
  111. exit 0
  112. ;;
  113. n) DRY_RUN=1
  114. echo " *** DRY RUN MODE ***"
  115. ;;
  116. u) USE_EXISTING=1
  117. echo " *** USE EXISTING DIRECTORIES MODE ***"
  118. ;;
  119. *)
  120. echo
  121. usage
  122. exit 1
  123. ;;
  124. esac
  125. done
  126. ###########################
  127. # Git worktrees to manage #
  128. ###########################
  129. COUNT=${#WORKTREE[@]}
  130. #############
  131. # Constants #
  132. #############
  133. # Control characters
  134. CNRM=$'\x1b[0;0m' # Clear color
  135. # Bright color
  136. BGRN=$'\x1b[1;32m'
  137. BBLU=$'\x1b[1;34m'
  138. BRED=$'\x1b[1;31m'
  139. BYEL=$'\x1b[1;33m'
  140. IWTH=$'\x1b[3;37m'
  141. # Strings for the pretty print.
  142. MARKER="${BBLU}[${BGRN}+${BBLU}]${CNRM}"
  143. SUCCESS="${BGRN}success${CNRM}"
  144. SKIPPED="${BYEL}skipped${CNRM}"
  145. FAILED="${BRED}failed${CNRM}"
  146. ####################
  147. # Helper functions #
  148. ####################
  149. # Validate the given returned value (error code), print success or failed. The
  150. # second argument is the error output in case of failure, it is printed out.
  151. # On failure, this function exits.
  152. function validate_ret
  153. {
  154. if [ "$1" -eq 0 ]; then
  155. printf "%s\\n" "$SUCCESS"
  156. else
  157. printf "%s\\n" "$FAILED"
  158. printf " %s\\n" "$2"
  159. exit 1
  160. fi
  161. }
  162. # Validate the given returned value (error code), print success, skipped, or
  163. # failed. If $USE_EXISTING is 0, fail on error, otherwise, skip on error.
  164. # The second argument is the error output in case of failure, it is printed
  165. # out. On failure, this function exits.
  166. function validate_ret_skip
  167. {
  168. if [ "$1" -ne 0 ]; then
  169. if [ "$USE_EXISTING" -eq "0" ]; then
  170. # Fail and exit with error
  171. validate_ret "$1" "$2 $USE_EXISTING_HINT"
  172. else
  173. printf "%s\\n" "$SKIPPED"
  174. printf " %s\\n" "${IWTH}$2${CNRM}"
  175. # Tell the caller to skip the rest of the function
  176. return 0
  177. fi
  178. fi
  179. # Tell the caller to continue
  180. return 1
  181. }
  182. # Create a directory, and any missing enclosing directories.
  183. # If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip.
  184. function make_directory
  185. {
  186. local cmd="mkdir -p '$1'"
  187. printf " %s Creating directory %s..." "$MARKER" "$1"
  188. local check_cmd="[ ! -d '$1' ]"
  189. msg=$( eval "$check_cmd" 2>&1 )
  190. if validate_ret_skip $? "Directory already exists."; then
  191. return
  192. fi
  193. if [ $DRY_RUN -eq 0 ]; then
  194. msg=$( eval "$cmd" 2>&1 )
  195. validate_ret $? "$msg"
  196. else
  197. printf "\\n %s\\n" "${IWTH}$cmd${CNRM}"
  198. fi
  199. }
  200. # Create a symlink from the first argument to the second argument
  201. # If the link already exists: fail if $USE_EXISTING is 0, otherwise skip.
  202. function make_symlink
  203. {
  204. local cmd="ln -s '$1' '$2'"
  205. printf " %s Creating symlink from %s to %s..." "$MARKER" "$1" "$2"
  206. local check_cmd="[ ! -e '$2' ]"
  207. msg=$( eval "$check_cmd" 2>&1 )
  208. if validate_ret_skip $? "File already exists."; then
  209. return
  210. fi
  211. if [ $DRY_RUN -eq 0 ]; then
  212. msg=$( eval "$cmd" 2>&1 )
  213. validate_ret $? "$msg"
  214. else
  215. printf "\\n %s\\n" "${IWTH}$cmd${CNRM}"
  216. fi
  217. }
  218. # Go into the directory or repository, even if $DRY_RUN is non-zero.
  219. # If the directory does not exist, fail and log an error.
  220. # Otherwise, silently succeed.
  221. function goto_dir
  222. {
  223. if ! cd "$1" 1>/dev/null 2>/dev/null ; then
  224. printf " %s Changing to directory %s..." "$MARKER" "$1"
  225. validate_ret 1 "$1: Not found. Stopping."
  226. fi
  227. }
  228. # Clone a repository into a directory.
  229. # If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip.
  230. function clone_repo
  231. {
  232. local cmd="git clone $TOR_EXTRA_CLONE_ARGS '$1' '$2'"
  233. printf " %s Cloning %s into %s..." "$MARKER" "$1" "$2"
  234. local check_cmd="[ ! -d '$2' ]"
  235. msg=$( eval "$check_cmd" 2>&1 )
  236. if validate_ret_skip $? "Directory already exists."; then
  237. # If we skip the clone, we need to do a fetch
  238. goto_dir "$ORIGIN_PATH"
  239. fetch_remote "origin"
  240. return
  241. fi
  242. if [ $DRY_RUN -eq 0 ]; then
  243. msg=$( eval "$cmd" 2>&1 )
  244. validate_ret $? "$msg"
  245. else
  246. printf "\\n %s\\n" "${IWTH}$cmd${CNRM}"
  247. fi
  248. }
  249. # Add a remote by name and URL.
  250. # If the remote already exists: fail if $USE_EXISTING is 0, otherwise skip.
  251. function add_remote
  252. {
  253. local cmd="git remote add '$1' '$2'"
  254. printf " %s Adding remote %s at %s..." "$MARKER" "$1" "$2"
  255. local check_cmd="git remote get-url '$1'"
  256. msg=$( eval "$check_cmd" 2>&1 )
  257. ret=$?
  258. # We don't want a remote, so we invert the exit status
  259. if validate_ret_skip $(( ! ret )) \
  260. "Remote already exists for $1 at $msg."; then
  261. return
  262. fi
  263. if [ $DRY_RUN -eq 0 ]; then
  264. msg=$( eval "$cmd" 2>&1 )
  265. validate_ret $? "$msg"
  266. else
  267. printf "\\n %s\\n" "${IWTH}$cmd${CNRM}"
  268. fi
  269. }
  270. # Set a remote's push URL by name and URL.
  271. function set_remote_push
  272. {
  273. local cmd="git remote set-url --push '$1' '$2'"
  274. printf " %s Setting remote %s push URL to '%s'..." "$MARKER" "$1" "$2"
  275. if [ $DRY_RUN -eq 0 ]; then
  276. msg=$( eval "$cmd" 2>&1 )
  277. validate_ret $? "$msg"
  278. else
  279. printf "\\n %s\\n" "${IWTH}$cmd${CNRM}"
  280. fi
  281. }
  282. # Fetch a remote by name.
  283. function fetch_remote
  284. {
  285. local cmd="git fetch '$1'"
  286. printf " %s Fetching %s..." "$MARKER" "$1"
  287. if [ $DRY_RUN -eq 0 ]; then
  288. msg=$( eval "$cmd" 2>&1 )
  289. validate_ret $? "$msg"
  290. else
  291. printf "\\n %s\\n" "${IWTH}$cmd${CNRM}"
  292. fi
  293. }
  294. # Replace the fetch configs for a remote with config if they match a pattern.
  295. function replace_fetch_config
  296. {
  297. local cmd="git config --replace-all remote.'$1'.fetch '$2' '$3'"
  298. printf " %s Replacing %s fetch configs for '%s'..." \
  299. "$MARKER" "$1" "$3"
  300. if [ $DRY_RUN -eq 0 ]; then
  301. msg=$( eval "$cmd" 2>&1 )
  302. validate_ret $? "$msg"
  303. else
  304. printf "\\n %s\\n" "${IWTH}$cmd${CNRM}"
  305. fi
  306. }
  307. # Set up the tor-github PR config, so tor-github/pr/NNNN/head points to GitHub
  308. # PR NNNN. In some repositories, "/head" is optional.
  309. function set_tor_github_pr_fetch_config
  310. {
  311. # Standard branches
  312. replace_fetch_config tor-github \
  313. "+refs/heads/*:refs/remotes/tor-github/*" \
  314. "refs/heads"
  315. # PRs
  316. replace_fetch_config "tor-github" \
  317. "+refs/pull/*:refs/remotes/tor-github/pr/*" \
  318. "refs/pull.*pr"
  319. }
  320. # Set up the tor-github PR config, so tor-gitlab/mr/NNNN points to GitHub
  321. # MR NNNN. In some repositories, "/head" is optional.
  322. function set_tor_gitlab_mr_fetch_config
  323. {
  324. # standard branches
  325. replace_fetch_config tor-gitlab \
  326. "+refs/heads/*:refs/remotes/tor-gitlab/*" \
  327. "refs/heads"
  328. # MRs
  329. replace_fetch_config tor-gitlab \
  330. "+refs/merge-requests/*/head:refs/remotes/tor-gitlab/mr/*" \
  331. "refs/merge-requests.*mr"
  332. }
  333. # Add a new worktree for branch at path.
  334. # If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip.
  335. function add_worktree
  336. {
  337. local cmd="git worktree add '$2' '$1'"
  338. printf " %s Adding worktree for %s at %s..." "$MARKER" "$1" "$2"
  339. local check_cmd="[ ! -d '$2' ]"
  340. msg=$( eval "$check_cmd" 2>&1 )
  341. if validate_ret_skip $? "Directory already exists."; then
  342. return
  343. fi
  344. if [ $DRY_RUN -eq 0 ]; then
  345. msg=$( eval "$cmd" 2>&1 )
  346. validate_ret $? "$msg"
  347. else
  348. printf "\\n %s\\n" "${IWTH}$cmd${CNRM}"
  349. fi
  350. }
  351. # Switch to the given branch name.
  352. # If the branch does not exist: fail.
  353. function switch_branch
  354. {
  355. local cmd="git checkout '$1'"
  356. printf " %s Switching branch to %s..." "$MARKER" "$1"
  357. if [ $DRY_RUN -eq 0 ]; then
  358. msg=$( eval "$cmd" 2>&1 )
  359. validate_ret $? "$msg"
  360. else
  361. printf "\\n %s\\n" "${IWTH}$cmd${CNRM}"
  362. fi
  363. }
  364. # Checkout a new branch with the given branch name.
  365. # If the branch already exists: fail if $USE_EXISTING is 0, otherwise skip.
  366. function new_branch
  367. {
  368. local cmd="git checkout -b '$1'"
  369. printf " %s Creating new branch %s..." "$MARKER" "$1"
  370. local check_cmd="git branch --list '$1'"
  371. msg=$( eval "$check_cmd" 2>&1 )
  372. if validate_ret_skip $? "Branch already exists."; then
  373. return
  374. fi
  375. if [ $DRY_RUN -eq 0 ]; then
  376. msg=$( eval "$cmd" 2>&1 )
  377. validate_ret $? "$msg"
  378. else
  379. printf "\\n %s\\n" "${IWTH}$cmd${CNRM}"
  380. fi
  381. }
  382. # Switch to an existing branch, or checkout a new branch with the given
  383. # branch name.
  384. function switch_or_new_branch
  385. {
  386. local cmd="git rev-parse --verify '$1'"
  387. if [ $DRY_RUN -eq 0 ]; then
  388. # Call switch_branch if there is a branch, or new_branch if there is not
  389. msg=$( eval "$cmd" 2>&1 )
  390. RET=$?
  391. if [ $RET -eq 0 ]; then
  392. # Branch: (commit id)
  393. switch_branch "$1"
  394. elif [ $RET -eq 128 ]; then
  395. # Not a branch: "fatal: Needed a single revision"
  396. new_branch "$1"
  397. else
  398. # Unexpected return value
  399. validate_ret $RET "$msg"
  400. fi
  401. else
  402. printf "\\n %s\\n" "${IWTH}$cmd${CNRM}, then depending on the result:"
  403. switch_branch "$1"
  404. new_branch "$1"
  405. fi
  406. }
  407. # Set the upstream for branch to upstream.
  408. function set_upstream
  409. {
  410. # Note the argument order is swapped
  411. local cmd="git branch --set-upstream-to='$2' '$1'"
  412. printf " %s Setting upstream for %s to %s..." "$MARKER" "$1" "$2"
  413. if [ $DRY_RUN -eq 0 ]; then
  414. msg=$( eval "$cmd" 2>&1 )
  415. validate_ret $? "$msg"
  416. else
  417. printf "\\n %s\\n" "${IWTH}$cmd${CNRM}"
  418. fi
  419. }
  420. ###############
  421. # Entry point #
  422. ###############
  423. printf "%s Setting up the repository and remote %s\\n" "$MARKER" \
  424. "${BYEL}origin${CNRM}"
  425. # First, fetch the origin.
  426. ORIGIN_PARENT=$(dirname "$ORIGIN_PATH")
  427. make_directory "$ORIGIN_PARENT"
  428. # This is just cd with an error check
  429. goto_dir "$ORIGIN_PARENT"
  430. # clone repository / origin remote
  431. clone_repo "$GIT_ORIGIN_PULL" "$TOR_MASTER_NAME"
  432. goto_dir "$ORIGIN_PATH"
  433. set_remote_push "origin" "$GIT_ORIGIN_PUSH"
  434. # upstream remote, if different to origin
  435. if [ "$DEFAULT_UPSTREAM_REMOTE" != "origin" ]; then
  436. printf "%s Setting up remote %s\\n" "$MARKER" \
  437. "${BYEL}$DEFAULT_UPSTREAM_REMOTE${CNRM}"
  438. add_remote "$DEFAULT_UPSTREAM_REMOTE" "$GIT_UPSTREAM_PULL"
  439. set_remote_push "$DEFAULT_UPSTREAM_REMOTE" "$GIT_UPSTREAM_PUSH"
  440. fetch_remote "$DEFAULT_UPSTREAM_REMOTE"
  441. fi
  442. # GitHub remote
  443. printf "%s Setting up remote %s\\n" "$MARKER" "${BYEL}tor-github${CNRM}"
  444. # Add remote
  445. add_remote "tor-github" "$GITHUB_PULL"
  446. set_remote_push "tor-github" "$GITHUB_PUSH"
  447. # Add custom fetch for PRs
  448. set_tor_github_pr_fetch_config
  449. # Now fetch them all
  450. fetch_remote "tor-github"
  451. # GitLab remote
  452. printf "%s Setting up remote %s\\n" "$MARKER" "${BYEL}tor-gitlab${CNRM}"
  453. add_remote "tor-gitlab" "$GITLAB_PULL"
  454. set_remote_push "tor-gitlab" "$GITLAB_PUSH"
  455. # Add custom fetch for MRs
  456. set_tor_gitlab_mr_fetch_config
  457. # Now fetch them all
  458. fetch_remote "tor-gitlab"
  459. # Extra remote
  460. if [ "$TOR_EXTRA_REMOTE_NAME" ]; then
  461. printf "%s Setting up remote %s\\n" "$MARKER" \
  462. "${BYEL}$TOR_EXTRA_REMOTE_NAME${CNRM}"
  463. # Add remote
  464. add_remote "$TOR_EXTRA_REMOTE_NAME" "$TOR_EXTRA_REMOTE_PULL"
  465. set_remote_push "$TOR_EXTRA_REMOTE_NAME" "$TOR_EXTRA_REMOTE_PUSH"
  466. # But leave it to the user to decide if they want to fetch it
  467. #fetch_remote "$TOR_EXTRA_REMOTE_NAME"
  468. fi
  469. # Go over all configured worktree.
  470. for ((i=0; i<COUNT; i++)); do
  471. branch=${!WORKTREE[$i]:0:1}
  472. repo_path=${!WORKTREE[$i]:1:1}
  473. printf "%s Handling branch %s\\n" "$MARKER" "${BYEL}$branch${CNRM}"
  474. # We cloned the repository, and master is the default branch
  475. if [ "$branch" = "master" ]; then
  476. if [ "$TOR_MASTER_NAME" != "master" ]; then
  477. # Set up a master link in the worktree directory
  478. make_symlink "$repo_path" "$GIT_PATH/$TOR_WKT_NAME/master"
  479. fi
  480. else
  481. # git makes worktree directories if they don't exist
  482. add_worktree "origin/$branch" "$repo_path"
  483. fi
  484. goto_dir "$repo_path"
  485. switch_or_new_branch "$branch"
  486. set_upstream "$branch" "origin/$branch"
  487. done
  488. echo
  489. echo "Remember to copy the git hooks from tor/scripts/git/*.git-hook to"
  490. echo "$ORIGIN_PATH/.git/hooks/*"