diff --git a/lib/configuration.sh b/lib/configuration.sh index a85184ab5..38958d703 100644 --- a/lib/configuration.sh +++ b/lib/configuration.sh @@ -29,6 +29,22 @@ ROOT_MAPPER="armbian-root" [[ -z $ROOTFS_TYPE ]] && ROOTFS_TYPE=ext4 # default rootfs type is ext4 [[ "ext4 f2fs btrfs nfs fel" != *$ROOTFS_TYPE* ]] && exit_with_error "Unknown rootfs type" "$ROOTFS_TYPE" +# check btrfs compression variable +if [[ -n $BTRFS_COMPRESSION ]];then + btrfs_compression_ok=0 + btrfs_cmp_chks="$(echo $BTRFS_COMPRESSION | sed -e 's/:[0-9]\+//')" + for c in lzo zlib zstd;do + [[ $c == $btrfs_cmp_chks ]] && btrfs_compression_ok=1 # ${BTRFS_COMPRESSION/:[0-9]+/} not works like zstd:12 + done + if [[ $btrfs_compression_ok -eq 0 ]] ;then + display_alert "wrong btrfs compression method" "$BTRFS_COMPRESSION" 'err' + display_alert "set BTRFS_COMPRESSION to default" "lzo" "wrn" + BTRFS_COMPRESSION=lzo + fi + unset btrfs_compression_ok + unset btrfs_cmp_chks +fi + # Fixed image size is in 1M dd blocks (MiB) # to get size of block device /dev/sdX execute as root: # echo $(( $(blockdev --getsize64 /dev/sdX) / 1024 / 1024 )) @@ -43,11 +59,12 @@ fi [[ $ROOTFS_TYPE == nfs ]] && FIXED_IMAGE_SIZE=64 # used by multiple sources - reduce code duplication -if [[ $USE_MAINLINE_GOOGLE_MIRROR == yes ]]; then - MAINLINE_KERNEL_SOURCE='https://kernel.googlesource.com/pub/scm/linux/kernel/git/stable/linux-stable' -else - MAINLINE_KERNEL_SOURCE='git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git' -fi +[[ $USE_MAINLINE_GOOGLE_MIRROR == yes ]] && MAINLINE_MIRROR=google +case $MAINLINE_MIRROR in + google) MAINLINE_KERNEL_SOURCE='https://kernel.googlesource.com/pub/scm/linux/kernel/git/stable/linux-stable' ;; + tuna) MAINLINE_KERNEL_SOURCE='https://mirrors.tuna.tsinghua.edu.cn/git/linux-stable.git' ;; + *) MAINLINE_KERNEL_SOURCE='git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git' ;; +esac MAINLINE_KERNEL_DIR='linux-mainline' if [[ $USE_GITHUB_UBOOT_MIRROR == yes ]]; then @@ -226,6 +243,11 @@ esac DEBIAN_MIRROR='httpredir.debian.org/debian' UBUNTU_MIRROR='ports.ubuntu.com/' +if [[ $DOWNLOAD_MIRROR == china ]] ; then + DEBIAN_MIRROR='mirrors.tuna.tsinghua.edu.cn/debian' + UBUNTU_MIRROR='mirrors.tuna.tsinghua.edu.cn/ubuntu-ports/' +fi + # For user override if [[ -f $SRC/userpatches/lib.config ]]; then display_alert "Using user configuration override" "userpatches/lib.config" "info" diff --git a/lib/debootstrap-ng.sh b/lib/debootstrap-ng.sh index acf8e783c..86a9a9996 100644 --- a/lib/debootstrap-ng.sh +++ b/lib/debootstrap-ng.sh @@ -296,7 +296,7 @@ prepare_partitions() mkopts[fat]='-n BOOT' mkopts[ext2]='-q' # mkopts[f2fs] is empty - # mkopts[btrfs] is empty + mkopts[btrfs]='--data single --metadata single --label btrfs' # mkopts[nfs] is empty mkfs[ext4]=ext4 @@ -310,7 +310,7 @@ prepare_partitions() # mountopts[ext2] is empty # mountopts[fat] is empty # mountopts[f2fs] is empty - mountopts[btrfs]=',commit=600,compress=lzo' + mountopts[btrfs]=",commit=600,compress=${BTRFS_COMPRESSION:-lzo}" # mountopts[nfs] is empty # stage: determine partition configuration @@ -358,7 +358,7 @@ prepare_partitions() case $ROOTFS_TYPE in btrfs) # Used for server images, currently no swap functionality, so disk space - # requirements are rather low since rootfs gets filled with compress-force=zlib + # requirements are rather low since rootfs gets filled with compress=$BTRFS_COMPRESSION local sdsize=$(bc -l <<< "scale=0; (($imagesize * 0.8) / 4 + 1) * 4") ;; *) @@ -375,7 +375,7 @@ prepare_partitions() # stage: create blank image display_alert "Creating blank image for rootfs" "$sdsize MiB" "info" - dd if=/dev/zero bs=1M status=none count=$sdsize | pv -p -b -r -s $(( $sdsize * 1024 * 1024 )) | dd status=none of=${SDCARD}.raw + truncate --size=${sdsize}M ${SDCARD}.raw # stage: calculate boot partition size local bootstart=$(($OFFSET * 2048)) @@ -433,8 +433,25 @@ prepare_partitions() display_alert "Creating rootfs" "$ROOTFS_TYPE on $rootdevice" mkfs.${mkfs[$ROOTFS_TYPE]} ${mkopts[$ROOTFS_TYPE]} $rootdevice [[ $ROOTFS_TYPE == ext4 ]] && tune2fs -o journal_data_writeback $rootdevice > /dev/null - [[ $ROOTFS_TYPE == btrfs ]] && local fscreateopt="-o compress-force=zlib" + + local btrfs_space_cache_version=v1 + local kernel_support_space_cache_v2=4.5 + dpkg --compare-versions $VER 'gt' $kernel_support_space_cache_v2 && btrfs_space_cache_version=v2 + [[ $ROOTFS_TYPE == btrfs ]] && local fscreateopt="-o compress=${BTRFS_COMPRESSION:-lzo},space_cache=${btrfs_space_cache_version}" + display_alert 'fscreateopt' "$fscreateopt" 'info' + unset btrfs_space_cache_version + unset kernel_support_space_cache_v2 + mount ${fscreateopt} $rootdevice $MOUNT/ + if [[ $ROOTFS_TYPE == btrfs ]];then + btrfs subvolume create $MOUNT/@$RELEASE + btrfs subvolume create $MOUNT/@home + btrfs subvolume create $MOUNT/@boot + umount $MOUNT + mount ${fscreateopt},subvol=@$RELEASE $rootdevice $MOUNT/ + mkdir "$MOUNT/home" + mount ${fscreateopt},subvol=@home $rootdevice $MOUNT/home + fi # create fstab (and crypttab) entry if [[ $CRYPTROOT_ENABLE == yes ]]; then # map the LUKS container partition via its UUID to be the 'cryptroot' device @@ -443,7 +460,13 @@ prepare_partitions() else local rootfs="UUID=$(blkid -s UUID -o value $rootdevice)" fi - echo "$rootfs / ${mkfs[$ROOTFS_TYPE]} defaults,noatime,nodiratime${mountopts[$ROOTFS_TYPE]} 0 1" >> $SDCARD/etc/fstab + + if [[ $ROOTFS_TYPE == btrfs ]]; then + echo "$rootfs / ${mkfs[$ROOTFS_TYPE]} defaults,noatime,nodiratime${mountopts[$ROOTFS_TYPE]},subvol=@$RELEASE 0 1" >> $SDCARD/etc/fstab + echo "$rootfs /home ${mkfs[$ROOTFS_TYPE]} defaults,noatime,nodiratime${mountopts[$ROOTFS_TYPE]},subvol=@home 0 1" >> $SDCARD/etc/fstab + else + echo "$rootfs / ${mkfs[$ROOTFS_TYPE]} defaults,noatime,nodiratime${mountopts[$ROOTFS_TYPE]} 0 1" >> $SDCARD/etc/fstab + fi fi if [[ -n $bootpart ]]; then display_alert "Creating /boot" "$bootfs" @@ -454,7 +477,7 @@ prepare_partitions() echo "UUID=$(blkid -s UUID -o value ${LOOP}p${bootpart}) /boot ${mkfs[$bootfs]} defaults${mountopts[$bootfs]} 0 2" >> $SDCARD/etc/fstab fi [[ $ROOTFS_TYPE == nfs ]] && echo "/dev/nfs / nfs defaults 0 0" >> $SDCARD/etc/fstab - echo "tmpfs /tmp tmpfs defaults,nosuid 0 0" >> $SDCARD/etc/fstab + echo "tmpfs /media tmpfs defaults,nosuid 0 0" >> $SDCARD/etc/fstab # stage: adjust boot script or boot environment if [[ -f $SDCARD/boot/armbianEnv.txt ]]; then @@ -492,6 +515,9 @@ prepare_partitions() fi fi + # kernel args for btrfs subvolume + [[ $ROOTFS_TYPE == btrfs ]] && echo "extraargs=rootflags=subvol=@$RELEASE" >> $SDCARD/boot/armbianEnv.txt + # recompile .cmd to .scr if boot.cmd exists [[ -f $SDCARD/boot/boot.cmd ]] && \ mkimage -C none -A arm -T script -d $SDCARD/boot/boot.cmd $SDCARD/boot/boot.scr > /dev/null 2>&1 @@ -536,7 +562,7 @@ create_image() if [[ $ROOTFS_TYPE != nfs ]]; then display_alert "Copying files to root directory" - rsync -aHWXh --exclude="/boot/*" --exclude="/dev/*" --exclude="/proc/*" --exclude="/run/*" --exclude="/tmp/*" \ + rsync -aHWXh --inplace --exclude="/boot/*" --exclude="/dev/*" --exclude="/proc/*" --exclude="/run/*" --exclude="/tmp/*" \ --exclude="/sys/*" --info=progress2,stats1 $SDCARD/ $MOUNT/ else display_alert "Creating rootfs archive" "rootfs.tgz" "info" @@ -548,10 +574,10 @@ create_image() display_alert "Copying files to /boot directory" if [[ $(findmnt --target $MOUNT/boot -o FSTYPE -n) == vfat ]]; then # fat32 - rsync -rLtWh --info=progress2,stats1 $SDCARD/boot $MOUNT + rsync -rLtWh --inplace --info=progress2,stats1 $SDCARD/boot $MOUNT else # ext4 - rsync -aHWXh --info=progress2,stats1 $SDCARD/boot $MOUNT + rsync -aHWXh --inplace --info=progress2,stats1 $SDCARD/boot $MOUNT fi # DEBUG: print free space diff --git a/lib/general.sh b/lib/general.sh index a3cd0f184..82fad1195 100644 --- a/lib/general.sh +++ b/lib/general.sh @@ -545,6 +545,8 @@ prepare_host() curl patchutils python liblz4-tool libpython2.7-dev linux-base swig libpython-dev aptly acl \ locales ncurses-base pixz dialog systemd-container udev lib32stdc++6 libc6-i386 lib32ncurses5 lib32tinfo5 \ bison libbison-dev flex libfl-dev cryptsetup gpgv1 gnupg1 cpio" + + which aria2c >/dev/null || hostdeps="$hostdeps aria2" local codename=$(lsb_release -sc) display_alert "Build host OS release" "${codename:-(unknown)}" "info" @@ -624,7 +626,7 @@ prepare_host() # sync clock if [[ $SYNC_CLOCK != no ]]; then display_alert "Syncing clock" "host" "info" - ntpdate -s ${NTP_SERVER:- time.ijs.si} + ntpdate -s ${NTP_SERVER:- pool.ntp.org} fi if [[ $(dpkg-query -W -f='${db:Status-Abbrev}\n' 'zlib1g:i386' 2>/dev/null) != *ii* ]]; then @@ -658,6 +660,9 @@ prepare_host() # Use backup server by default to balance the load ARMBIANSERVER=dl.armbian.com + if [[ $DOWNLOAD_MIRROR == 'china' ]]; then + ARMBIANSERVER='mirrors.tuna.tsinghua.edu.cn/armbian-releases' + fi local toolchains=( "https://${ARMBIANSERVER}/_toolchains/gcc-linaro-aarch64-none-elf-4.8-2013.11_linux.tar.xz" @@ -671,7 +676,6 @@ prepare_host() "https://${ARMBIANSERVER}/_toolchains/gcc-linaro-5.5.0-2017.10-x86_64_arm-linux-gnueabihf.tar.xz" "https://${ARMBIANSERVER}/_toolchains/gcc-linaro-6.4.1-2017.11-x86_64_arm-linux-gnueabihf.tar.xz" "https://${ARMBIANSERVER}/_toolchains/gcc-linaro-6.4.1-2017.11-x86_64_aarch64-linux-gnu.tar.xz" - "https://${ARMBIANSERVER}/_toolchains/gcc-linaro-7.4.1-2019.02-x86_64_aarch64-linux-gnu.tar.xz" "https://${ARMBIANSERVER}/_toolchains/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabihf.tar.xz" "https://${ARMBIANSERVER}/_toolchains/gcc-linaro-7.4.1-2019.02-x86_64_arm-eabi.tar.xz" "https://${ARMBIANSERVER}/_toolchains/gcc-linaro-7.4.1-2019.02-x86_64_arm-linux-gnueabi.tar.xz" @@ -718,6 +722,16 @@ prepare_host() fi } +# downloader() +downloader() +{ + [ $# -ne 2 ] && exit_with_error "downloader args count" "$#" + args='--file-allocation=falloc --auto-file-renaming=false --continue=true --allow-overwrite=false' + display_alert "downloader args 1" "$1" "info" + display_alert "downloader args 2" "$2" "info" + aria2c $args "$1" -o "$2" +} + # download_toolchain # download_toolchain() @@ -733,8 +747,8 @@ download_toolchain() cd $SRC/cache/toolchains/ display_alert "Downloading" "$dirname" - curl -Lf --progress-bar $url -o $filename - curl -Lf --progress-bar ${url}.asc -o ${filename}.asc + downloader $url $filename + downloader ${url}.asc ${filename}.asc local verified=false @@ -783,7 +797,7 @@ download_etcher_cli() cd $SRC/cache/utility/ display_alert "Downloading" "$dirname" - curl -Lf --progress-bar $url -o $filename + downloader $url $filename local verified=false local b=$(sha256sum $filename) diff --git a/packages/extras/firmware.sh b/packages/extras/firmware.sh index 43de2c065..13c9ea4da 100644 --- a/packages/extras/firmware.sh +++ b/packages/extras/firmware.sh @@ -14,14 +14,17 @@ build_firmware() local plugin_repo="https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git" local plugin_dir="armbian-firmware${FULL}" [[ -d $SRC/cache/sources/$plugin_dir ]] && rm -rf $SRC/cache/sources/$plugin_dir + mkdir -p $SRC/cache/sources/$plugin_dir/lib/firmware fetch_from_repo "https://github.com/armbian/firmware" "armbian-firmware-git" "branch:master" if [[ -n $FULL ]]; then - fetch_from_repo "$plugin_repo" "$plugin_dir/lib/firmware" "branch:master" + fetch_from_repo "$plugin_repo" "linux-firmware-git" "branch:master" + # cp : create hardlinks + cp -alf $SRC/cache/sources/linux-firmware-git/* $SRC/cache/sources/$plugin_dir/lib/firmware/ fi - mkdir -p $SRC/cache/sources/$plugin_dir/lib/firmware # overlay our firmware - cp -R $SRC/cache/sources/armbian-firmware-git/* $SRC/cache/sources/$plugin_dir/lib/firmware + # cp : create hardlinks + cp -alf $SRC/cache/sources/armbian-firmware-git/* $SRC/cache/sources/$plugin_dir/lib/firmware/ # cleanup what's not needed for sure rm -rf $SRC/cache/sources/$plugin_dir/lib/firmware/{amdgpu,amd-ucode,radeon,nvidia,matrox,.git} @@ -43,9 +46,9 @@ build_firmware() cd $SRC/cache/sources # pack - mv armbian-firmware${FULL} armbian-firmware${FULL}_${REVISION}_all + ln -s armbian-firmware${FULL} armbian-firmware${FULL}_${REVISION}_all fakeroot dpkg -b armbian-firmware${FULL}_${REVISION}_all >> $DEST/debug/install.log 2>&1 - mv armbian-firmware${FULL}_${REVISION}_all armbian-firmware${FULL} + rm armbian-firmware${FULL}_${REVISION}_all mv armbian-firmware${FULL}_${REVISION}_all.deb $DEST/debs/ || display_alert "Failed moving firmware package" "" "wrn" }