#!/bin/sh die () { echo "error: $*" >&2 exit 1 } all () { # pp maybe finish upd -a first and start to suspend, and it cause insp music() failed to rsync to pp, so I need to finish rsync to ssh before pp finish upd, so I put music() here, not perfect but good enough [ "$hostname" = xyzinsp ] && music # monthly_misc() needs to use qbittorrent-nox APIs, so I need to restart qbt before that, so I put qb() at top here, not perfect but good enough if [ "$hostname" = xyzib ]; then qb fi # After linux upgrade, if not reboot, plug in external HDD will not be # detected, not sure about USD thumb drive tho. monthly_misc() needs to # plug in USD thumb drive, so monthly_misc() needs to run before pac() # which is in fast(). monthly_misc fast clean # don't run reflector if it is pp or insp, because they all VPN to ba now if ! { [ "$hostname" = xyzpp ] || [ "$hostname" = xyzinsp ];}; then refl fi if [ "$hostname" = xyzinsp ] || [ "$hostname" = xyzpp ]; then userjs fi } backup () { #case "$hostname" in # xyzinsp) backup_branch=master;; # *) backup_branch=${hostname#xyz};; #esac if [ "$hostname" = xyzinsp ]; then # Pull repos and branches to insp, so one more backup on insp or duplicity will backup to ib # Also sync the local dir to work on it cd "$HOME/programs/config_local_arch" || exit git branch | awk '{print ($1=="*")?$2:$1}' | while read -r branch; do git checkout "$branch" git pull # Push any branches other than master/pp to codeberg or any future alternative website for mirroring. Needed for public codes/data, considering my death may result in all copies on my computers being lost. case "$branch" in master|pp) ;; *) git push;; esac done # always go back to a specific branch, because future new branshes may # make previous git checkout checkout to that new branch cause rsync # have to sync that new branch instead git checkout master # git checkout will change mtime, need to change back so rsync won't think it needs to backup these files # https://stackoverflow.com/q/1964470/9008720 # https://stackoverflow.com/q/21735435/9008720 # https://github.com/MestreLion/git-tools # aur/git-tools-git git restore-mtime ( umask 077 # need to pull VPS cfgs repos (ca, ib, ba, etc.) to /root/programs # for insp to duplicity backup to ca and ib to satisfy 321 backup # rule for dir in $(sudo find /root/programs -maxdepth 1 -mindepth 1 -type d -name 'config_local_arch_secrets*'); do sudo -E git -C "$dir" pull done ) # rsync backup from ib to insp ~/backup/ib # Note: don't use things like `:.config/qBittorrent` because it will sync # to /home/xyz/backup/ib/.config, which is not what I want. I want # /home/xyz/backup/ib/home/xyz/.config rsync -avPR --delete xyz@ib.flylightning.xyz:/home/xyz/.config/qBittorrent :/home/xyz/.local/share/qBittorrent/BT_backup "$HOME/backup/ib" # rsync backup from pp to insp ~/backup/pp rsync -vPu pp:.config/myconf/upd_rsync_files "$XDG_CONFIG_HOME/myconf/upd_rsync_files_pp" # --files-from make -a not imply -r, so need to specify explicitly rsync -avPRr --delete --files-from="$XDG_CONFIG_HOME/myconf/upd_rsync_files_pp" pp:/ "$HOME/backup/pp" # backup ib to pp, not much storage needed and I don't want to use duplicity to backup to ca yet again rsync -avP --delete "$HOME/backup/ib" pp:backup # duplicity backup to ib # https://wiki.archlinux.org/title/Duplicity # Need // for absolute path, see manpage URL FORMAT section. If not use //, will store to /home/xyz/home/xyz/... # --files-from has a bug, this bug makes source url that is / not working while /home works, more see vq notes # --use-agent not working when ssh to pp and insp, works on insp, not sure why # --use-agent maybe timeout on gpg key and failed when do full backup, maybe due to key stored in gpg agent timeout, so I'm not using --use-agent on insp now sudo duplicity --ssh-askpass --encrypt-key 9790577D2BE328D46838117ED3F54FE03F3C68D6 --sign-key 05899270DF25BB1EEDF57BE824F769E5D08C9E9A --full-if-older-than 2Y --include /etc/.cfgs --include /root/archive --include /root/programs --include-filelist "/home/xyz/.config/myconf/upd_rsync_files" --exclude / / "sftp://xyz@ib.flylightning.xyz//home/xyz/backup/master" # duplicity backup to ca sudo duplicity --ssh-askpass --encrypt-key 9790577D2BE328D46838117ED3F54FE03F3C68D6 --sign-key 05899270DF25BB1EEDF57BE824F769E5D08C9E9A --full-if-older-than 2Y --include /etc/.cfgs --include /root/archive --include /root/programs --include-filelist "/home/xyz/.config/myconf/upd_rsync_files" --exclude / / "sftp://xyz@ca.flylightning.xyz//home/xyz/backup/master" fi if [ "$hostname" = xyzpp ]; then # duplicity backup to ib sudo duplicity --ssh-askpass --encrypt-key 9790577D2BE328D46838117ED3F54FE03F3C68D6 --sign-key 05899270DF25BB1EEDF57BE824F769E5D08C9E9A --full-if-older-than 2Y --include /etc/.cfgs --include-filelist "/home/xyz/.config/myconf/upd_rsync_files" --exclude / / "sftp://xyz@ib.flylightning.xyz//home/xyz/backup/pp" # pp no need duplicity backup to ca, because 321 backup rule is already # satisfied: pp files in ~/.config/myconf/upd_rsync_files are backed up # in pp, insp, and ib duplicity; pp /etc/.cfgs is backed up in pp, insp # gitolite, and ib duplicity. fi } clean () { if [ "$hostname" = xyzinsp ]; then nsxiv -c # my ways # -exec can't replace -execdir here find "$XDG_CACHE_HOME/nsxiv/" -depth -type d -empty -execdir rmdir -- '{}' \+ # -exec can replace -execdir here #find "$XDG_CACHE_HOME/nsxiv/" -depth -type d -execdir rmdir --ignore-fail-on-non-empty -- '{}' \+ # nsxiv man page way #find "$XDG_CACHE_HOME/nsxiv/" -depth -type d -empty ! -name '.' -exec rmdir -- '{}' \; fi if [ "$hostname" = xyzinsp ] || [ "$hostname" = xyzpp ]; then cd "$HOME/.mozilla/firefox/xxxxxxxx.fly/prefsjs_backups" || exit # https://stackoverflow.com/a/34862475/9008720 ls -t | tail -n +11 | tr '\n' '\0' | xargs -0 rm -- cd "$HOME/.mozilla/firefox/xxxxxxxx.fly/userjs_backups" || exit ls -t | tail -n +11 | tr '\n' '\0' | xargs -0 rm -- # https://unix.stackexchange.com/questions/92095/reset-atq-list-to-zero sudo systemctl stop atd echo 0 | sudo tee /var/spool/atd/.SEQ > /dev/null sudo systemctl start atd rm -rf "$XDG_VIDEOS_DIR/recordings/tmp/" fi paru -aSc --noconfirm # https://wiki.archlinux.org/title/Pacman/Tips_and_tricks#Removing_unused_packages_(orphans) pacman -Qdttq | sudo pacman -Rns --noconfirm - # https://wiki.archlinux.org/title/Pacman/Tips_and_tricks#Detecting_more_unneeded_packages # remove special orphan pkgs: dependency cycle, excessive denpendencies etc. pacman -Qqd | sudo pacman -Rsu --noconfirm - } # basic daily stuff fast () { # After linux upgrade, if not reboot, plug in external HDD will not be # detected. backup() needs to plugin external HDD, so backup() needs to run # before pac(). Now, I do not use an external HDD for frequent bakcup any # more, but I still would like to put it in this order to consider in the # future I maybe use one again. if [ "$hostname" = xyzinsp ] || [ "$hostname" = xyzpp ]; then backup fi pac misc } userjs () { kill $(pidof "$BROWSER") # change working dir for cleaner cd "$HOME/.mozilla/firefox/xxxxxxxx.fly" || exit arkenfox-cleaner -s # when multiple firefox profiles, it will prompt me to choose, which breaks automation, so I explicitly specify one profile arkenfox-updater -s -p "$HOME/.mozilla/firefox/xxxxxxxx.fly" } misc () { nv= "$EDITOR" +PlugClean! +PlugUpdate +qa if [ "$hostname" = xyzinsp ] || [ "$hostname" = xyzpp ]; then tldr --update fi if [ "$hostname" = xyzinsp ]; then # nvchecker pkgctl updated pkgbuilds. Need test, commit, push, and aurpublish. # Test shows pkgctl version upgrade need 2>&1, else awk not work, not # sure why, maybe related to pkgctl's pretty outputs. # Test shows awk need $3 instead of $2, not sure why. nv="$(pkgctl version upgrade "$HOME/programs/repos/aur/stardict-wikt-en-all" \ "$HOME/programs/repos/aur/stardict-wikt-en-en" 2>&1 \ | tee /dev/tty \ | awk -F'[: ]' '/upgraded/{printf(" %s",$3)}')" log="${log}nvchecker:$nv " sudo hardcode-fixer ssh pp '[ -s "$HOME/.local/share/sdcv_history" ] && cat "$HOME/.local/share/sdcv_history" && rm "$HOME/.local/share/sdcv_history"' >> "$XDG_DATA_HOME/sdcv_history" awk '!a[$0]++' "$XDG_DATA_HOME/sdcv_history" | sponge "$XDG_DATA_HOME/sdcv_history" # temperory solution before find a way of using git submodule or subtree with `cfg -l` git -C "$HOME/.mozilla/firefox/xxxxxxxx.fly/chrome/firefox-csshacks" pull git -C "$XDG_DOCUMENTS_DIR/notes" commit --all -m 'update' git -C "$XDG_DOCUMENTS_DIR/notes" push git -C "$HOME/programs/reminders" commit --all -m 'update' git -C "$HOME/programs/reminders" push pass git push argospm update rsync -vPu "$HOME/.abook/addressbook" pp:.abook/addressbook rsync -avP --delete "$XDG_DOCUMENTS_DIR/ids" pp:documents rsync -vPu "$HOME/programs/roff/myresume/resume_chinese.odt" \ "$HOME/programs/roff/myresume/resume_chinese.pdf" \ pp:programs/roff/myresume rsync -avP --delete "$XDG_DOCUMENTS_DIR/books/novels" pp:documents/books fi if [ "$hostname" = xyzpp ]; then git -C "$XDG_DOCUMENTS_DIR/notes" pull git -C "$HOME/programs/reminders" pull git -C "$HOME/programs/roff/myresume" pull pass git pull fi } pac () { pacout="$(sudo pacman --noconfirm -Syu | tee /dev/tty)" pacpacs="$(echo "$pacout" | grep -m1 '^Packages' | cut -d' ' -f3-)" # Update rust toolchains before paru so paru can compile things in newest rust if needed. [ "$hostname" = xyzinsp ] && rustup update aurout="$(paru --color never -aSu | tee /dev/tty)" aurpacs="$(echo "$aurout" | grep '^Aur' | cut -d' ' -f3-)" [ "$hostname" = xyzpp ] && sudo flatpak update -y # /usr/share/libalpm/hooks/rebuild-detector.hook has a line `NeedsTargets` shows it maybe checkrebuild only upgraded packages by `printf 'zoom\nminiconda3' | checkrebuild` instead of maybe check all by `checkrebuild`, so I think query pacman hook will be faster than run another `checkrebuild` # notes about awk f=1 things see https://git.flylightning.xyz/public_archive_codes/tree/sh/mrt # about `/^(\(|:|=)/ {f=0}`: # - consider $aurout start with `^:: Looking for devel upgrades...` , rebuild-detector.hook maybe the last hook to run for $pacout # - consider ^(4/5), the hook is not the last # - consider paru `==> Making package: ...`, the hook maybe followed by this. Note: paru somehow still gives color output even if I use --color never, so I can't check with ^=, so I choose to check with ==> # awk use `if(!a[$2]++)` to check if package name is repeated in multiple checkrebuild pacman hook run, happened when upgrade python cause all python packages need to be rebuilt # TODO: Some packages maybe are rebuilt later on when paru upgrade packages, but those will still got shown in upd log. Try consider this situation. e.g., when pacman upgrade packages, checkrebuild hook output a b c d packages, then paru upgrade d, now checkrebuild hook output a b c, the final upd log will have a b c d all packages instead of a b c checkrebuild_pacs="$(echo "$pacout$aurout" | awk ' /^\([0-9]+\/[0-9]+\) Checking which packages need to be rebuilt$/ {f=1; next} /^(\(|:)|==>/ {f=0} f { if($2!~"miniconda3") if(!a[$2]++) printf("%s ",$2) }')" # part steal from aur comment # sometimes "ERROR: Failure while downloading": https://github.com/neovim/neovim/issues/15709 # echo 1, printf 1 and yes 1 all works? not sure why # aur neovim-nightly-bin has some issue on 12/26/2021? switch to community repo neovim temporary #rm -rf ~/.cache/paru/clone/neovim-nightly-bin/ && echo 1 | PARU_PAGER=cat paru --rebuild --redownload neovim-nightly-bin if [ "$hostname" = xyzinsp ]; then case "$pacpacs" in *qt5-base*) echo 1 | PARU_PAGER=cat paru --rebuild qt5-styleplugins;; esac case "$pacpacs" in *qt6-base*) echo 1 | PARU_PAGER=cat paru --rebuild qt6gtk2;; esac fi pacman -Qqme > "$XDG_CONFIG_HOME/myconf/pacman_Qqme" pacman -Qqne > "$XDG_CONFIG_HOME/myconf/pacman_Qqne" systemctl list-unit-files --state=enabled > "$XDG_CONFIG_HOME/myconf/sye" systemctl --user list-unit-files --state=enabled > "$XDG_CONFIG_HOME/myconf/syue" # pacdiff default use pacman database, so no need `sudo -E` for find, but will be a little bit slower # [^-] consider util-linux; \(^\|[^-]\) consider linux is the first package, ex: pacout is only 'linux-6.6.6' log="${log}updated pacman packages: $pacpacs updated aur packages: $aurpacs pacdiff: $(pacdiff -o | tr '\n' ' ') checkrebuild: $checkrebuild_pacs $(if echo "$pacpacs" | grep -q '\(^\|[^-]\)linux-\(megi-\)\?[0-9]'; then echo 'kernel upgraded, need reboot'; fi) " } qb () { sudo systemctl stop qbittorrent-nox@xyz.service find "$XDG_DATA_HOME/qBittorrent/nova3/engines" -maxdepth 1 -type f ! -name 'jackett*' -a ! -name '__init__.py' -delete curlqb "$XDG_DATA_HOME/qBittorrent/nova3/engines" sudo systemctl start qbittorrent-nox@xyz.service } refl () { # why not use http: # https://www.reddit.com/r/archlinux/comments/kx149z/should_i_use_http_mirrors/ # https://www.reddit.com/r/archlinux/comments/ej4k4d/is_it_safe_to_use_not_secured_http_pacman_mirrors/ # rsync may need to change XferCommand in /etc/pacman.conf # https://www.reddit.com/r/archlinux/comments/mynw6e/rsync_mirrors_with_pacman/ # need --delay so no super out of sync mirrors case "$hostname" in xyzib) sudo reflector --verbose --save /etc/pacman.d/mirrorlist --country ro --protocol https --delay 1 --fastest 3;; *) sudo reflector --verbose --save /etc/pacman.d/mirrorlist --country us --protocol https --delay 1 --latest 25 --score 25 --fastest 10;; esac } music () { yt-dlp -f 'bestaudio[ext=opus]/bestaudio' --match-filter 'license=cc-by' --match-filter 'title~=(?i)cc-by' -P "$XDG_MUSIC_DIR/cc-by/scott_buckley" https://soundcloud.com/scottbuckley/tracks rsync -avP --delete "$XDG_MUSIC_DIR/cc-by/scott_buckley" pp:music/cc-by rsync -avP --delete "$XDG_MUSIC_DIR/gpl-3.0-only" pp:music rsync -avP --delete "$XDG_MUSIC_DIR/favorite" pp:music } monthly_misc () { if [ "$hostname" = xyzinsp ]; then sudo scp xyz@ba.flylightning.xyz:/etc/pacman.d/mirrorlist /etc/pacman.d/mirrorlist # Monthly backup gpg key, ssh key, ~/.password-store, and notes for # backup diaster recovery in case both my insp laptop and pp phone are # lost. I prefer monthly backup those instead of backup those more # frequently, because plug in and unplug drives more frequently is # tedious and things backed up in bd2 does not need to be very recent. while ! [ -d /run/media/xyz/bd2 ]; do alarm 0 'Plug in bd2 usb flash drive' echo 'Plug in bd2 usb flash drive' sleep 10 done git -C /run/media/xyz/bd2/backup/notes pull git -C /run/media/xyz/bd2/backup/.password-store pull # consider password change for the keys and I forget to backup the keys # encrypted with new password # # these two files are not directory so I think --delete is unnecessary rsync -avP "$XDG_DOCUMENTS_DIR/paperwork/sec_pgp_key.asc" "$HOME/.ssh/id_rsa" /run/media/xyz/bd2/backup # bd2 is the label name of the unlocked ext4 fs, it was configured with # sth. like `sudo e2label /dev/mapper/luks-... bd2`. ebd2 is the label # name of the locked crypto_LUKS fs, it was configured with `sudo # cryptsetup config /dev/sdx --label ebd2`. # # Another way to represent disk is using uuid. ebd2 locked crypto_LUKS # fs uuid is 0c2fceb2-9743-4872-99f7-8d87d53c9270, so I can use # /dev/disk/by-uuid/0c2fceb2-9743-4872-99f7-8d87d53c9270 to represent # it. bd2 unlocked ext4 fs uuid is # f7da70df-94a1-478d-98a2-86af8be54c56, so I can use # /dev/disk/by-uuid/f7da70df-94a1-478d-98a2-86af8be54c56 for it, I can # also use /dev/mapper/luks-0c2fceb2-9743-4872-99f7-8d87d53c9270 for # bd2, note here 0c2fceb2-9743-4872-99f7-8d87d53c9270 is the ebd2 # locked crypto_LUKS fs uuid. However, udisksctl does not recognize # /dev/mapper/luks-..., but recognize /dev/disk/...; cryptosetup close # and umount recognize both. More ways see dirs in /dev/disk, and see # output of press tab after `udiskiectl info -b`. # # Note `udisksctl lock` need ebd2 locked crypto_LUKS fs, but `sudo # cryptsetup close` need bd2 unlocked ext4 fs. umount /run/media/xyz/bd2 udisksctl lock -b /dev/disk/by-label/ebd2 udisksctl power-off -b /dev/disk/by-label/ebd2 alarm 0 'Unplug bd2 usb flash drive' # https://github.com/qbittorrent/qBittorrent/wiki#webui-api curl -sS -X POST 'ibwg:57151/api/v2/search/updatePlugins' & # in the past, I have both stwg and ibwg, now only one # another way: `jq -r '.[]|"\(.added_on)\t\(.hash)\t\(.name)"'` curl -sS 'ibwg:57151/api/v2/torrents/info?category=useful' | jq -r '.[]|[.added_on,.hash,.name]|@tsv' | grep 'archlinux.*\.iso' | sort -n | head -n-1 | awk '{print $2}' | while read -r hash; do # need POST to delete torrents. -d default POST, so no need `-X POST` curl -sS -d "hashes=$hash" -d deleteFiles=true 'ibwg:57151/api/v2/torrents/delete' done & wait while ! [ -d /run/media/xyz/Ventoy ]; do alarm 0 'Plug in ventoy usb flash drive' echo 'Plug in ventoy usb flash drive' sleep 10 done #[ -d /run/media/xyz/Ventoy ] || die "No usb flash drive" rsync -vPu ib:downloads/torrents/useful/archlinux*.iso /run/media/xyz/Ventoy/archlinux-x86_64.iso # only check checksum and gpg signature on insp is sufficient for me, else too much work if curl -sS -o /run/media/xyz/Ventoy/archlinux-x86_64.iso.sig https://mirror.fcix.net/archlinux/iso/latest/archlinux-x86_64.iso.sig; then gpg --verify /run/media/xyz/Ventoy/archlinux-x86_64.iso.sig || die 'Arch iso gpg signature check failed' else die 'Arch iso gpg signature download failed' fi # need to cd to iso file dir to checksum cd /run/media/xyz/Ventoy || exit curl -sS https://mirror.fcix.net/archlinux/iso/latest/sha256sums.txt | grep archlinux-x86_64\.iso | tee /run/media/xyz/Ventoy/archlinux-x86_64.iso.sha256 | sha256sum -c || die 'Arch iso checksum does not match' # if stay at /run/media/xyz/Ventoy, will cause it be busy and can't be umount, which will cause `ventoy -u` fail # need to be after `wait`, because checksum need to be at ventoy dir cd || exit # busybox df if filesystem name too long, it will put the filesystem # name in a separate line which nornally is in the second line and # there's total three lines. So I use df -P option which seems busybox # will also not put filesystem name in a separate line and behave same # as gnu df. disk="$(df -P /run/media/xyz/Ventoy/ | awk 'END{sub(/[[:digit:]]+$/,"",$1);print $1}')" # another way: #disk="$(realpath /dev/disk/by-label/Ventoy | awk 'END{sub(/[[:digit:]]+$/,"",$0);print}')" sudo ventoy -l "$disk" | awk '/Ventoy:/{a=$2} /Ventoy Version in Disk:/{b=$NF;exit} END{exit((a==b)?1:0)}' && echo y | sudo ventoy -u "$disk" umount /run/media/xyz/Ventoy /run/media/xyz/FAT32DIR udisksctl power-off -b /dev/disk/by-label/Ventoy alarm 0 'Unplug ventoy usb flash drive' fi } hostname="$(hostname)" [ "$hostname" = xyzpp ] && gsettings set org.gnome.settings-daemon.plugins.power sleep-inactive-ac-type nothing if [ $# -eq 0 ]; then fast else while getopts abcfjmMopqr opt; do case $opt in a)all;; b)backup;; c)clean;; f)fast;; j)userjs;; m)misc;; M)music;; o)monthly_misc;; p)pac;; q)qb;; r)refl;; \?)exit 1;; esac done fi [ "$log" ] && printf '\n%s' "$log" | tee "$XDG_DOCUMENTS_DIR/logs/upd.log" [ "$hostname" = xyzpp ] && gsettings set org.gnome.settings-daemon.plugins.power sleep-inactive-ac-type suspend # `[ "$hostname" = xyzpp ] && ...` if check failed will have exit status of 1, unlike check with `if` # I decided to always `exit 0` if reached end, so commands like `upd -p && ...` can keep running exit 0