Merge pull request #631 from htm-community/ci_artefacts

CI fetch artifacts

.circleci/config.yml

@@ -134,9 +134,14 @@ jobs:
             python setup.py bdist_wheel
             cd build/scripts
             make package
+            mkdir -p /tmp/uploads
+            cp -a *.tar.gz /tmp/uploads
 
+      # API for storing and downloading artifacts to/from CI:
+      # https://circleci.com/docs/2.0/artifacts/
       - store_artifacts:
-          path: dist/*.whl
+          path: /tmp/uploads
+
       ##- persist_to_workspace:
       ##    root: dist
       ##    paths:
@@ -190,10 +195,14 @@ jobs:
 
 workflows:
   version: 2
+
   osx-build-test-deploy: #runs on each commit
     jobs:
       - osx-build-debug    # Mac OS/X raw ("build-debug")
-      - osx-build-release  # Mac OS/X raw ("build-release")
+      - osx-build-release:  # Mac OS/X raw ("build-release")
+          requires:
+            - osx-build-debug # all C++ debug tests must pass before Release (and thus artifact) is created
+
 
   nightly: # build and tests taking long time (currecntly ARM64 build ~5hrs), runs only once a day
     triggers:

.travis.yml

@@ -25,6 +25,7 @@ matrix:
             - cmake-data
             - xsltproc #only for doc transform to html , can be removed
             - valgrind
+            - wget # for downloading artifacts
       env:
         - MATRIX_EVAL="CC=gcc-8 && CXX=g++-8"
         - PLATFORM="linux"
@@ -59,13 +60,16 @@ before_deploy:
   - python $TRAVIS_BUILD_DIR/setup.py bdist_wheel
   - echo "Generating binary release:"
   - make package
+  - ../../ci/fetch-artifacts.sh
+
 
 deploy:
 ## Github releases
+# API: https://docs.travis-ci.com/user/deployment/releases/
   provider: releases
   api_key: "$GITHUB_TOKEN"
   file_glob: true # allows '*' in file below
-  file: "$TRAVIS_BUILD_DIR/build/scripts/*.tar.gz"
+  file: "$TRAVIS_BUILD_DIR/build/scripts/*.{tar.gz,zip}"
   skip_cleanup: true
   name: "testing" # GH Release name, undocumented option
   on:

appveyor.yml

@@ -132,9 +132,9 @@ after_build:
 # Disable automatic tests
 test: off
 
-#artifacts:
-#  # Non-recursive search in build folder for Wheels
-#  - path: '%HTM_CORE%\build\scripts\*.zip'
+artifacts:
+  # Non-recursive search in build folder for Wheels
+  - path: 'build\scripts\*.zip'
 
 # NOTE: This is turned off and will likely be removed once deployments of
 #       releases are controlled from a central authority. -- Matt
@@ -149,20 +149,20 @@ test: off
 #         twine upload -u $env:PYPI_USERNAME -p $env:PYPI_PASSWORD -r pypi $env:HTM_CORE\bindings\py\dist\htm.bindings-$env:BINDINGS_VERSION-cp27-none-$env:wheel_name_suffix.whl
 #       }
 
-deploy:
-## GitHub
-# Deploy to GitHub Releases
-# see https://www.appveyor.com/docs/deployment/github/
-#
-  release: 'testing' #htmcore-win64-v$(appveyor_build_version)
-  description: 'HTM.core binary release for Windows (64bit)' #desc is mandatory
-  provider: GitHub
-  auth_token: 
-    secure: "%GITHUB_TOKEN%"
-  artifact: '%HTM_CORE%\build\scripts\*.zip' # upload .zip from 'make package'
-  draft: false
-  prerelease: false
-  force_update: true #override files in existing release
-  on:
-    branch: master                # release from master branch only
-    appveyor_repo_tag: true       # deploy on tag push only
+    #deploy:
+    ### GitHub
+    ## Deploy to GitHub Releases
+    ## see https://www.appveyor.com/docs/deployment/github/
+    ##
+    #  release: 'testing' #htmcore-win64-v$(appveyor_build_version)
+    #  description: 'HTM.core binary release for Windows (64bit)' #desc is mandatory
+    #  provider: GitHub
+    #  auth_token: 
+    #    secure: "%GITHUB_TOKEN%"
+    #  artifact: '\build\scripts\*.zip' # upload .zip from 'make package'
+    #  draft: false
+    #  prerelease: false
+    #  force_update: true #override files in existing release
+    #  on:
+    #    branch: master                # release from master branch only
+    #    appveyor_repo_tag: true       # deploy on tag push only

ci/fetch-artifacts.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# script to fetch artifacts from Win,OSX CI to Travis,
+# where all 3 files are deployed to GitHub for release. 
+
+WINDOWS_ARTIFACT_URL="https://ci.appveyor.com/api/buildjobs/by4cpqmul79tp9ag/artifacts/build/scripts/htm_core-6866bd13ece35608313709e82c29b2260ee7be73-windows64.zip" #FIXME make the URLs automated, needs API?
+OSX_ARTIFACT_URL="https://3238-118031881-gh.circle-artifacts.com/0/tmp/uploads/htm_core-7b7b53c798b3795088ce772e96c6994068fc1302-darwin64.tar.gz"
+
+echo "Fetching Windows artifacts:" 
+retry=false
+for i in `seq 10`; do
+  echo "Downloading artifacts. Try $i"
+  wget "$WINDOWS_ARTIFACT_URL" || retry=true
+  if [[ x"$retry" == "xtrue" ]]; then
+    echo "sleep for 5min is good"
+    sleep 300 # wait 5min to retry. Other CI needs to finish its build
+    retry=false
+  else
+    echo "happy"
+    ls *.zip
+    ls *.gz
+    break
+  fi
+done
+
+echo "Fetching OSX artifacts:"
+retry=false
+for i in `seq 10`; do
+  echo "Downloading artifacts. Try $i"
+  wget "$OSX_ARTIFACT_URL" || retry=true
+  if [[ x"$retry" == "xtrue" ]]; then
+    echo "sleep for 5min is good"
+    sleep 300 # wait 5min to retry. Other CI needs to finish its build
+    retry=false
+  else
+    echo "happy"
+    ls *.zip
+    ls *.gz
+    break
+  fi
+done