squashed history

.github/workflows/debian-packages-build.yml

@@ -0,0 +1,184 @@
+name: Build test and deploy ZFS packages for Debian 10, Ubuntu 18.04 and later
+on:
+  push:
+    branches:    
+      - master
+    paths-ignore:
+    - 'README.md'
+jobs:
+  build:
+    name: Building packages
+    runs-on: ubuntu-18.04
+    steps:
+      - name: clone zfs upstream repo
+        shell: bash
+        run: |
+          mkdir ~/zfs && cd ~/zfs
+          git init
+          git fetch https://github.com/openzfs/zfs.git refs/heads/master
+          git checkout FETCH_HEAD
+      - name: clone our zfs ubuntu repo
+        shell: bash
+        run: |
+          echo "ref triggered = ${{ github.ref }}"
+          mkdir ~/zfs/debian && cd ~/zfs/debian
+          git init
+          git fetch https://github.com/${{ github.repository }}.git ${{ github.ref }} --depth=1
+          git checkout FETCH_HEAD
+      - name: retrieve build packages
+        shell: bash
+        run: |
+          sudo DEBIAN_FRONTEND=noninteractive apt-get -yq update
+          sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install dpkg-dev alien autoconf automake build-essential dkms fakeroot gawk gdebi-core libacl1-dev libaio-dev libattr1-dev libblkid-dev libdevmapper-dev libelf-dev libselinux-dev libssl-dev libtool libudev-dev nfs-kernel-server python3 python3-dev python3-cffi python3-setuptools uuid-dev zlib1g-dev linux-headers-$(uname -r) python3-all-dev python3-sphinx
+      - name: build zfs packages
+        shell: bash
+        run: |
+          echo "current folder"
+          pwd
+          cd ~/zfs
+          ls ./debian/changelog
+          dpkg-buildpackage -b -rfakeroot -us -uc --no-sign
+          mkdir ./zfs-deb-packages
+          cp ../*.deb ./zfs-deb-packages
+          ls ./zfs-deb-packages -al
+      - name: cache created packages
+        uses: actions/cache@v1
+        with:
+          path: ~/zfs/zfs-deb-packages
+          key: ${{ runner.OS }}-zfs-${{ github.sha }}
+  test:
+    needs: build
+    name: Testing packages
+    runs-on: ubuntu-18.04
+    strategy:
+      fail-fast: true
+      matrix:
+        testbatch:
+          - 'alloc_class,limits,arc,channel_program,cp_files,deadman,btree,cache,history,hkdf,inuse,zfs_property,zfs_receive,zfs_reservation,zfs_send,zfs_set,zfs_share,zfs_snapshot,zfs_unload-key,zfs_unmount,zfs_unshare,zfs_upgrade,zpool,zpool_destroy,zpool_detach'
+          - 'rootpool,scrub_mirror,pool_checkpoint'
+          - 'acl,atime,bootfs,cachefile,casenorm,chattr,checksum,clean_mirror,compression,ctime,delegate,devices,events,exec,fault,features,grow_pool'
+          - 'grow_replicas,mv_files,cli_user,zfs_mount,zfs_promote,zfs_rollback,zpool_events,zpool_expand,zpool_export,zpool_add,zpool_attach,zpool_clear,zpool_create'
+          - 'zpool_get,zpool_history,zpool_import,zpool_labelclear,zpool_offline,zpool_online'
+          - 'zpool_reopen,zpool_remove,zpool_replace,zpool_scrub,zpool_set,zpool_status,zpool_sync,zpool_upgrade,redacted_send,redundancy'
+          - 'zfs_destroy,large_files,largest_pool,link_count,migration,inheritance,refquota,refreserv,rename_dirs,replacement,log_spacemap,persist_l2arc,reservation'
+          - 'zfs,zfs_bookmark,zfs_change-key,zfs_clone,zfs_copies,zfs_create,zfs_diff,zfs_get,zfs_inherit,zfs_load-key,zfs_rename,mmap,mmp,mount,nestedfs,no_space,nopwrite,online_offline,pool_names,poolversion'
+          - 'zdb,mmp'
+          - 'privilege,quota,raidz,redundancy,rsend'
+          - 'slog,snapshot,snapused,sparse,threadsappend,tmpfile,truncate,upgrade,userquota,vdev_zaps,write_dirs,xattr,zvol,libzfs'
+    steps:
+      - name: Restore cached packages
+        uses: actions/cache@v1
+        with:
+          path: ./zfs-deb-packages
+          key: ${{ runner.OS }}-zfs-${{ github.sha }}
+      - name: setup local repo and put earlier-built ZFS packages there
+        shell: bash
+        run: |
+          cd ./zfs-deb-packages
+          ls ./ -alt
+          pwd
+          sudo bash -c "dpkg-scanpackages -m . > Packages"
+          sudo bash -c "echo 'deb [trusted=yes] file:$(pwd) ./' >> /etc/apt/sources.list"
+          sudo bash -c "echo -e \"Package: *\nPin: origin \\\"\\\"\nPin-Priority: 1001\" > /etc/apt/preferences.d/90_zfs"
+          sudo DEBIAN_FRONTEND=noninteractive apt-get -yq update
+      - name: install earlier-built ZFS packages along with packages required for conducting tests
+        shell: bash
+        run: |
+          sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install linux-modules-extra-$(uname -r)
+          sudo bash -c "echo \"zfs-dkms zfs-dkms/note-incompatible-licenses note true\" | debconf-set-selections"
+          sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install zfs-dkms zfsutils-linux zfs-zed zfs-test parted lsscsi ksh attr acl fio quota nfs-kernel-server samba-common-bin linux-tools-generic linux-tools-$(uname -r) linux-cloud-tools-$(uname -r)
+          zfs version
+      - name: run tests
+        shell: bash
+        run: |
+          sudo rm -rf /etc/hostid
+          /usr/share/zfs/zfs-tests.sh -v -T ${{ matrix.testbatch }}
+      - name: show test output on failure
+        if: failure()
+        shell: bash
+        run: |
+          find /var/tmp/test_results/current/log -type f -name '*' -printf "%f\n" -exec cut -c -800 {} \;
+  deploy:
+    needs: test
+    name: Deploying artefacts to debian repository
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Restore cached packages
+        uses: actions/cache@v1
+        with:
+          path: ./zfs-deb-packages
+          key: ${{ runner.OS }}-zfs-${{ github.sha }}
+      - name: adding required packages for repo publishing
+        shell: bash
+        run: |
+          sudo bash -c "echo 'deb http://repo.aptly.info/ squeeze main' >> /etc/apt/sources.list"
+          wget -qO - https://www.aptly.info/pubkey.txt | sudo apt-key add -
+          sudo DEBIAN_FRONTEND=noninteractive apt-get -yq update
+          sudo DEBIAN_FRONTEND=noninteractive apt-get -yq install gnupg aptly
+      - name: create repo and add packages
+        shell: bash
+        run: |
+          aptly repo create -distribution="zfs-debian-experimental" -component="main" zfs-debian-experimental
+          sed -i 's/\"architectures\": \[\]/\"architectures\": \[\"amd64\"\]/g' ~/.aptly.conf
+          echo -n "${{ secrets.GPG_KEY }}" > ./apt-private-key.asc
+          echo ${{ secrets.GPG_PASS }} | gpg --batch --import ./apt-private-key.asc
+          rm -rf ./apt-private-key.asc
+          ls ./zfs-deb-packages/*.deb -1 | xargs aptly repo add zfs-debian-experimental
+          aptly publish repo --gpg-key="$(gpg --list-public-keys --keyid-format 0xLONG | sed -n -E 's/^pub\s+\w{1,30}[0-9]+\/0x([0-9A-Z]+)\s+.+$/\1/p')" --batch=true --passphrase="${{ secrets.GPG_PASS }}" zfs-debian-experimental
+          rm -rf ./zfs-deb-packages
+      - uses: actions/checkout@v2
+        with:
+          ref: gh-pages
+      - name: push updates
+        shell: bash
+        run: |2
+          git config --global user.email "$(git log -n 1 --format=format:%ae)"
+          git config --global user.name "$(git log -n 1 --format=format:%an)"
+          rm ./public -rf
+          cp -r /home/runner/.aptly/public ./
+          export PAGE_FOOTER_DATE_TAG=$(date -Is)
+          sed -i -E "s/(<span class=\"datetag\">)[^<]*(<\/span>)/\1${PAGE_FOOTER_DATE_TAG}\2/" ./index.html
+          python - <<END
+          import os
+          indexTextStart = """<!DOCTYPE html>
+          <html>
+          <head><title>Index of {folderPath}</title></head>
+          <body>
+              <h2>Index of {folderPath}</h2>
+              <hr>
+              <ul>
+                          <li>
+                        <a href='../'>../</a>
+                 </li>
+          """
+          indexTextEnd = """
+                  </ul>
+          <hr/>
+          <p>Page generated: ${PAGE_FOOTER_DATE_TAG}</p>
+          </body>
+          </html>
+          """
+          def index_folder(folderPath):
+              print("Indexing: " + folderPath +'/')
+              files = os.listdir(folderPath)
+              root = folderPath
+              if folderPath == '.':
+                      root = 'Root'
+              indexText = indexTextStart.format(folderPath=root)
+              for file in files:
+                      if file != 'index.html':
+                              indexText += "\t\t<li>\n\t\t\t<a href='" + file + "'>" + file + "</a>\n\t\t</li>\n"
+                      if os.path.isdir(folderPath+'/'+file):
+                              index_folder(folderPath + '/' + file)
+              indexText += indexTextEnd
+              index = open(folderPath+'/index.html', "w")
+              index.write(indexText)
+          index_folder('./public')
+          END
+          rm -rf ./.git
+          git init
+          git checkout --orphan "gh-pages"
+          git remote add origin "https://x-access-token:${{ secrets.GITHUB_PERSONAL_TOKEN }}@github.com/${{ github.repository }}.git"
+          git add --all
+          git commit -am "automatic pages update from ${{ github.ref }} commit ${{ github.sha }}"
+          git push --force --set-upstream origin gh-pages

NEWS

@@ -0,0 +1,38 @@
+zfs-linux (0.8.0-1) experimental; urgency=medium
+
+  ZFS 0.8.0 introduced the following NEW features:
+
+    * Native encryption
+    * Raw encrypted 'zfs send/receive'
+    * Device removal
+    * Pool checkpoints
+    * Pool TRIM
+    * Pool initialization
+    * Project accounting and quota
+    * Channel programs
+    * Pyzfs
+    * Python 3 compatibility
+    * Direct IO
+
+  And the following performance improvements:
+
+    * Sequential scrub and resilver
+    * Allocation classes
+    * Administrative commands
+    * Parallel allocation
+    * Deferred resilvers
+    * ZFS Intent Log (ZIL)
+    * Volumes
+    * QAT
+
+  Details about the features and performance improvements can be found at:
+
+    https://github.com/zfsonlinux/zfs/releases/tag/zfs-0.8.0
+
+  NEWS on the Debian packaging side:
+
+    * ZFS doesn't depend on the spl-dkms package anymore, because
+      upstream had merged SPL source into ZFS source tree, and that
+      module is now included in the zfs-dkms package.
+
+ -- Mo Zhou <[email protected]>  Fri, 24 May 2019 03:27:43 +0000

README.md

@@ -0,0 +1,46 @@
+![img](http://zfsonlinux.org/images/zfs-linux.png)
+
+This repository is intended to help ZFS enthusiasts to build, test and deploy Debian packages for ZFS on Linux, 
+using latest master branch from [official repo](https://github.com/openzfs/zfs) and patches to it, listed in patches/series folder. Official distributions are often lag behing dev progress, so, those wanting to use latest "bleeding edge" features, can leverage this possibility.
+
+The repository is based on excellent work of [Debian ZFS team](https://salsa.debian.org/zfsonlinux-team/zfs.git) with a few customizations, allowed to build run smoontly on Github workflow runners. 
+
+Attached to this repository exist a Github actions workflow located [here](./.github/workflows/debian-packages-build.yml), which upon each master commit, checks out [ZFS upstream repo](https://github.com/openzfs/zfs) 
+then applies patches, builds, tests and upon tests success, deploys  built and tested debian binary packages into tiny, yet fully functional [APT repository](https://terem42.github.io/zfs-debian/), created using Github Pages.
+
+You can later reference this repository in your deployments or fork this repo (insructions below) and build your custom version of packages, having all the beeding edge changes you want. This approach gives you complete freedom over the building 
+version and applied patches over the source code, while still being guided by master branch from official repo.
+If you want to change upstream repo source or branch from offcial repo to your forked repo, just edit .github/workflow/ubuntu-packages-build.yml workflow file. 
+
+# Official Resources
+
+  * [Site](http://zfsonlinux.org)
+  * [Wiki](https://github.com/zfsonlinux/zfs/wiki)
+  * [Mailing lists](https://github.com/zfsonlinux/zfs/wiki/Mailing-Lists)
+  * [OpenZFS site](http://open-zfs.org/)
+
+# Installation
+
+Add the signing key for this repo:
+
+````
+sudo bash -c "wget -O - https://terem42.github.io/zfs-debian/apt_pub.gpg | apt-key add -"
+````
+
+Next, install packages:
+
+````
+sudo add-apt-repository 'deb https://terem42.github.io/zfs-debian/public zfs-debian-experimental main'
+sudo bash -c "echo \"zfs-dkms zfs-dkms/note-incompatible-licenses note true\" | debconf-set-selections"
+sudo apt install -t zfs-debian-experimental --yes zfs-dkms zfsutils-linux  
+````
+# List of tests skipped at Github runners
+I've added patch, adding several tests to the "known to skip or fail" list, see it [here](./patches/terem-0002-added-known-to-skip-tests-on-github-workflow-vm.patch). Check it for complete list of these tests. If you want to use self-hosted runners, do not forget to exclude this patch from ./patches/series file to have complete test coverage.
+
+# Forking the repo
+Fork and setup this repo for personal ZFS packages tailored to your needs in 5 easy steps:
+ * create github secret GITHUB_PERSONAL_TOKEN containing github personal acess token, it is nesessary for pushing changes into gh-pages branch after succesfull build and test (for security reasons, github.token used by Gihtub Actions workflow does not trigger properly Github Pages site rebuild after push to gh-pages branch).
+ * either import your own or generate GPG key (it is used by github workflow to sign packages, read [here](https://help.github.com/en/github/authenticating-to-github/generating-a-new-gpg-key) on how to create gpg key, afterwards export public key into apt_pub.gpg file an commit changes into gh-pages branch
+ * export as ASCII-armored your private GPG key into file, then create another Github secret GPG_KEY with the file contents
+ * create github secret GPG_PASS holding your password for the private key
+ * edit ./index.html file within gh-pages branch to match your repo and info you want, it will hold your repo entry page, explaining details on how to use this repo