build: add azure pipelines build

.gitattributes

@@ -1,3 +1,7 @@
 # Ignore generated files in GitHub language statistics and diffs
 /MANUAL.* linguist-generated=true
 /rclone.1 linguist-generated=true
+
+# Don't fiddle with the line endings of test data
+**/testdata/** -text
+**/test/** -text

Makefile

@@ -1,5 +1,5 @@
 SHELL = bash
-BRANCH := $(or $(APPVEYOR_REPO_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
+BRANCH := $(or $(APPVEYOR_REPO_BRANCH),$(TRAVIS_BRANCH),$(BUILD_SOURCEBRANCHNAME),$(shell git rev-parse --abbrev-ref HEAD))
 LAST_TAG := $(shell git describe --tags --abbrev=0)
 ifeq ($(BRANCH),$(LAST_TAG))
 	BRANCH := master
@@ -17,7 +17,10 @@ ifneq ($(TAG),$(LAST_TAG))
 endif
 GO_VERSION := $(shell go version)
 GO_FILES := $(shell go list ./... | grep -v /vendor/ )
-BETA_PATH := $(BRANCH_PATH)$(TAG)
+ifdef BETA_SUBDIR
+	BETA_SUBDIR := /$(BETA_SUBDIR)
+endif
+BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR)
 BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
 BETA_UPLOAD_ROOT := memstore:beta-rclone-org
 BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
@@ -168,14 +171,14 @@ endif
 	@echo Beta release ready at $(BETA_URL)/testbuilds
 
 travis_beta:
-ifeq ($(TRAVIS_OS_NAME),linux)
+ifeq (linux,$(filter linux,$(subst Linux,linux,$(TRAVIS_OS_NAME) $(AGENT_OS))))
 	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*\.tar.gz'
 endif
 	git log $(LAST_TAG).. > /tmp/git-log.txt
 	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(TAG)
 	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
 ifndef BRANCH_PATH
-	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)
+	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
 endif
 	@echo Beta release ready at $(BETA_URL)
 

azure-pipelines.yml

@@ -0,0 +1,231 @@
+---
+# Azure pipelines build for rclone
+# Parts stolen shamelessly from all round the Internet, especially Caddy
+# -*- compile-command: "yamllint -f parsable azure-pipelines.yml" -*-
+
+trigger:
+  branches:
+    include:
+      - '*'
+  tags:
+    include:
+      - '*'
+
+variables:
+  GOROOT: $(gorootDir)/go
+  GOPATH: $(system.defaultWorkingDirectory)/gopath
+  GOCACHE: $(system.defaultWorkingDirectory)/gocache
+  GOBIN: $(GOPATH)/bin
+  modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)'
+  GO111MODULE: 'off'
+  GOTAGS: cmount
+  GO_LATEST: false
+  CPATH: ''
+  GO_INSTALL_ARCH: amd64
+
+strategy:
+  matrix:
+    linux:
+      imageName: ubuntu-16.04
+      gorootDir: /usr/local
+      GO_VERSION: latest
+      GOTAGS: cmount
+      BUILD_FLAGS: '-include "^linux/"'
+      MAKE_CHECK: true
+      MAKE_QUICKTEST: true
+      DEPLOY: true
+    mac:
+      imageName: macos-10.13
+      gorootDir: /usr/local
+      GO_VERSION: latest
+      GOTAGS: ""  # cmount doesn't work on osx travis for some reason
+      BUILD_FLAGS: '-include "^darwin/" -cgo'
+      MAKE_QUICKTEST: true
+      MAKE_RACEQUICKTEST: true
+      DEPLOY: true
+    windows_amd64:
+      imageName: windows-2019
+      gorootDir: C:\
+      GO_VERSION: latest
+      BUILD_FLAGS: '-include "^windows/amd64" -cgo'
+      MAKE_QUICKTEST: true
+      DEPLOY: true
+    windows_386:
+      imageName: windows-2019
+      gorootDir: C:\
+      GO_VERSION: latest
+      GO_INSTALL_ARCH: 386
+      BUILD_FLAGS: '-include "^windows/386" -cgo'
+      MAKE_QUICKTEST: true
+      DEPLOY: true
+    other_os:
+      imageName: ubuntu-16.04
+      gorootDir: /usr/local
+      GO_VERSION: latest
+      BUILD_FLAGS: '-exclude "^(windows|darwin|linux)/"'
+      MAKE_COMPILE_ALL: true
+      DEPLOY: true
+    modules_race:
+      imageName: ubuntu-16.04
+      gorootDir: /usr/local
+      GO_VERSION: latest
+      GO111MODULE: on
+      GOPROXY: https://proxy.golang.org
+      MAKE_QUICKTEST: true
+      MAKE_RACEQUICKTEST: true
+    go1.9:
+      imageName: ubuntu-16.04
+      gorootDir: /usr/local
+      GOCACHE: ''               # build caching only came in go1.10
+      GO_VERSION: go1.9.7
+      MAKE_QUICKTEST: true
+    go1.10:
+      imageName: ubuntu-16.04
+      gorootDir: /usr/local
+      GO_VERSION: go1.10.8
+      MAKE_QUICKTEST: true
+    go1.11:
+      imageName: ubuntu-16.04
+      gorootDir: /usr/local
+      GO_VERSION: go1.11.12
+      MAKE_QUICKTEST: true
+
+pool:
+  vmImage: $(imageName)
+
+steps:
+  - bash: |
+      latestGo=$(curl "https://golang.org/VERSION?m=text")
+      echo "##vso[task.setvariable variable=GO_VERSION]$latestGo"
+      echo "##vso[task.setvariable variable=GO_LATEST]true"
+      echo "Latest Go version: $latestGo"
+    condition: eq( variables['GO_VERSION'], 'latest' )
+    displayName: "Get latest Go version"
+
+  - bash: |
+      sudo rm -f $(which go)
+      echo '##vso[task.prependpath]$(GOBIN)'
+      echo '##vso[task.prependpath]$(GOROOT)/bin'
+      mkdir -p '$(modulePath)'
+      shopt -s extglob
+      shopt -s dotglob
+      mv !(gopath) '$(modulePath)'
+    displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH
+
+  - task: CacheBeta@0
+    continueOnError: true
+    inputs:
+      key: go-build-cache | "$(Agent.JobName)"
+      path: $(GOCACHE)
+    displayName: Cache go build
+    condition: ne( variables['GOCACHE'], '' )
+
+  # Install Libraries (varies by platform)
+
+  - bash: |
+      sudo modprobe fuse
+      sudo chmod 666 /dev/fuse
+      sudo chown root:$USER /etc/fuse.conf
+      sudo apt-get install fuse libfuse-dev rpm pkg-config
+    condition: eq( variables['Agent.OS'], 'Linux' )
+    displayName: Install Libraries on Linux
+
+  - bash: |
+      brew update
+      brew tap caskroom/cask
+      brew cask install osxfuse
+    condition: eq( variables['Agent.OS'], 'Darwin' )
+    displayName: Install Libraries on macOS
+
+  - powershell: |
+      $ProgressPreference = 'SilentlyContinue'
+      choco install -y winfsp zip
+      Write-Host "##vso[task.setvariable variable=CPATH]C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse"
+      if ($env:GO_INSTALL_ARCH -eq "386") {
+        choco install -y mingw --forcex86 --force
+        Write-Host "##vso[task.prependpath]C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw32\\bin"
+      }
+      # Copy mingw32-make.exe to make.exe so the same command line
+      # can be used on Windows as on macOS and Linux
+      $path = (get-command mingw32-make.exe).Path
+      Copy-Item -Path $path -Destination (Join-Path (Split-Path -Path $path) 'make.exe')
+    condition: eq( variables['Agent.OS'], 'Windows_NT' )
+    displayName: Install Libraries on Windows
+
+
+  # Install Go (this varies by platform)
+
+  - bash: |
+      wget "https://dl.google.com/go/$(GO_VERSION).linux-$(GO_INSTALL_ARCH).tar.gz"
+      sudo mkdir $(gorootDir)
+      sudo chown ${USER}:${USER} $(gorootDir)
+      tar -C $(gorootDir) -xzf "$(GO_VERSION).linux-$(GO_INSTALL_ARCH).tar.gz"
+    condition: eq( variables['Agent.OS'], 'Linux' )
+    displayName: Install Go on Linux
+
+  - bash: |
+      wget "https://dl.google.com/go/$(GO_VERSION).darwin-$(GO_INSTALL_ARCH).tar.gz"
+      sudo tar -C $(gorootDir) -xzf "$(GO_VERSION).darwin-$(GO_INSTALL_ARCH).tar.gz"
+    condition: eq( variables['Agent.OS'], 'Darwin' )
+    displayName: Install Go on macOS
+
+  - powershell: |
+      $ProgressPreference = 'SilentlyContinue'
+      Write-Host "Downloading Go $(GO_VERSION) for $(GO_INSTALL_ARCH)"
+      (New-Object System.Net.WebClient).DownloadFile("https://dl.google.com/go/$(GO_VERSION).windows-$(GO_INSTALL_ARCH).zip", "$(GO_VERSION).windows-$(GO_INSTALL_ARCH).zip")
+      Write-Host "Extracting Go"
+      Expand-Archive "$(GO_VERSION).windows-$(GO_INSTALL_ARCH).zip" -DestinationPath "$(gorootDir)"
+    condition: eq( variables['Agent.OS'], 'Windows_NT' )
+    displayName: Install Go on Windows
+
+  # Display environment for debugging
+
+  - bash: |
+      printf "Using go at: $(which go)\n"
+      printf "Go version: $(go version)\n"
+      printf "\n\nGo environment:\n\n"
+      go env
+      printf "\n\nRclone environment:\n\n"
+      make vars
+      printf "\n\nSystem environment:\n\n"
+      env
+    workingDirectory: '$(modulePath)'
+    displayName: Print Go version and environment
+
+  # Run Tests
+
+  - bash: |
+      make
+      make quicktest
+    workingDirectory: '$(modulePath)'
+    displayName: Run tests
+    condition: eq( variables['MAKE_QUICKTEST'], 'true' )
+
+  - bash: |
+      make racequicktest
+    workingDirectory: '$(modulePath)'
+    displayName: Race test
+    condition: eq( variables['MAKE_RACEQUICKTEST'], 'true' )
+
+  - bash: |
+      make build_dep
+      make check
+    workingDirectory: '$(modulePath)'
+    displayName: Code quality test
+    condition: eq( variables['MAKE_CHECK'], 'true' )
+
+  - bash: |
+      make
+      make compile_all
+    workingDirectory: '$(modulePath)'
+    displayName: Compile all architectures test
+    condition: eq( variables['MAKE_COMPILE_ALL'], 'true' )
+
+  - bash: |
+      make travis_beta
+    env:
+      RCLONE_CONFIG_PASS: $(RCLONE_CONFIG_PASS)
+      BETA_SUBDIR: 'azure_pipelines'  # FIXME remove when removing travis/appveyor
+    workingDirectory: '$(modulePath)'
+    displayName: Deploy built binaries
+    condition: and( eq( variables['DEPLOY'], 'true' ), ne( variables['Build.Reason'], 'PullRequest' ) )