diff --git a/appveyor.yml b/appveyor.yml
index 88e130e9e9..7d60aa6399 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -4,15 +4,36 @@ branches:
   except:
     - gh-pages
 
-init:
-  - ps: (get-psprovider 'FileSystem').Home = $(pwd)
-  - ps: Install-PackageProvider Nuget -Force
-  - ps: Install-Module -Name Pester -Repository PSGallery -Force
-  - ps: Install-Module -Name PSScriptAnalyzer -Repository PSGallery -Force
-
 build: off
+deploy: off
+clone_depth: 50
+
+image:
+- Visual Studio 2017
+
+environment:
+  matrix:
+    - PowerShell: 5
+    - PowerShell: 6
+
+cache:
+  - '%USERPROFILE%\Documents\WindowsPowerShell\Modules -> appveyor.yml'
+
+matrix:
+  fast_finish: true
 
-test_script:
-  - ps: $result = invoke-pester -path test/ -outputfile TestResults.xml -outputformat NUnitXML -passthru; $env:failedcount = $result.failedcount
-  - ps: (new-object net.webclient).uploadfile("https://ci.appveyor.com/api/testresults/nunit/$($env:APPVEYOR_JOB_ID)", (resolve-path ./TestResults.xml))
-  - ps: if($env:failedcount -gt 0) { exit $env:failedcount }
+for:
+- matrix:
+    only:
+      - PowerShell: 5
+  install:
+    - ps: .\test\bin\init.ps1
+  test_script:
+    - ps: .\test\bin\test.ps1
+- matrix:
+    only:
+      - PowerShell: 6
+  install:
+    - pwsh: .\test\bin\init.ps1
+  test_script:
+    - pwsh: .\test\bin\test.ps1
diff --git a/test/Scoop-Alias.Tests.ps1 b/test/Scoop-Alias.Tests.ps1
index d8e454693d..028309d326 100644
--- a/test/Scoop-Alias.Tests.ps1
+++ b/test/Scoop-Alias.Tests.ps1
@@ -2,7 +2,7 @@
 
 reset_aliases
 
-describe "add_alias" {
+describe "add_alias" -Tag 'Scoop' {
   mock shimdir { "TestDrive:\shim" }
   mock set_config { }
   mock get_config { @{} }
diff --git a/test/Scoop-Config.Tests.ps1 b/test/Scoop-Config.Tests.ps1
index ace7718c0e..0e043d614c 100644
--- a/test/Scoop-Config.Tests.ps1
+++ b/test/Scoop-Config.Tests.ps1
@@ -1,7 +1,7 @@
 . "$psscriptroot\..\lib\core.ps1"
 . "$psscriptroot\..\lib\config.ps1"
 
-describe "hashtable" {
+describe "hashtable" -Tag 'Scoop' {
     $json = '{ "one": 1, "two": [ { "a": "a" }, "b", 2 ], "three": { "four": 4 }, "five": true, "six": false, "seven": "\/Date(1529917395805)\/" }'
 
     it "converts pscustomobject to hashtable" {
diff --git a/test/Scoop-Core.Tests.ps1 b/test/Scoop-Core.Tests.ps1
index e6dcc6bb97..ac5588aaf4 100644
--- a/test/Scoop-Core.Tests.ps1
+++ b/test/Scoop-Core.Tests.ps1
@@ -6,7 +6,7 @@
 $repo_dir = (Get-Item $MyInvocation.MyCommand.Path).directory.parent.FullName
 $isUnix = is_unix
 
-describe "is_directory" {
+describe "is_directory" -Tag 'Scoop' {
     beforeall {
         $working_dir = setup_working "is_directory"
     }
@@ -23,7 +23,7 @@ describe "is_directory" {
     }
 }
 
-describe "movedir" {
+describe "movedir" -Tag 'Scoop' {
     $extract_dir = "subdir"
     $extract_to = $null
 
@@ -61,7 +61,7 @@ describe "movedir" {
     }
 }
 
-describe "unzip_old" {
+describe "unzip_old" -Tag 'Scoop' {
     beforeall {
         $working_dir = setup_working "unzip_old"
     }
@@ -117,7 +117,7 @@ describe "unzip_old" {
     }
 }
 
-describe "shim" {
+describe "shim" -Tag 'Scoop' {
     beforeall {
         $working_dir = setup_working "shim"
         $shimdir = shimdir
@@ -153,7 +153,7 @@ describe "shim" {
     }
 }
 
-describe "rm_shim" {
+describe "rm_shim" -Tag 'Scoop' {
     beforeall {
         $working_dir = setup_working "shim"
         $shimdir = shimdir
@@ -172,7 +172,7 @@ describe "rm_shim" {
     }
 }
 
-Describe "get_app_name_from_ps1_shim" {
+Describe "get_app_name_from_ps1_shim" -Tag 'Scoop' {
     BeforeAll {
         $working_dir = setup_working "shim"
         $shimdir = shimdir
@@ -211,7 +211,7 @@ Describe "get_app_name_from_ps1_shim" {
     }
 }
 
-describe "ensure_robocopy_in_path" {
+describe "ensure_robocopy_in_path" -Tag 'Scoop' {
     $shimdir = shimdir $false
     mock versiondir { $repo_dir }
 
@@ -245,7 +245,7 @@ describe "ensure_robocopy_in_path" {
     }
 }
 
-describe 'sanitary_path' {
+describe 'sanitary_path' -Tag 'Scoop' {
   it 'removes invalid path characters from a string' {
     $path = 'test?.json'
     $valid_path = sanitary_path $path
@@ -254,7 +254,7 @@ describe 'sanitary_path' {
   }
 }
 
-describe 'app' {
+describe 'app' -Tag 'Scoop' {
     it 'parses the bucket name from an app query' {
         $query = "C:\test.json"
         $app, $bucket, $version = parse_app $query
diff --git a/test/Scoop-GetOpts.Tests.ps1 b/test/Scoop-GetOpts.Tests.ps1
index 98525b21ef..7796defbeb 100644
--- a/test/Scoop-GetOpts.Tests.ps1
+++ b/test/Scoop-GetOpts.Tests.ps1
@@ -1,7 +1,7 @@
 . "$psscriptroot\Scoop-TestLib.ps1"
 . "$psscriptroot\..\lib\getopt.ps1"
 
-describe "getopt" {
+describe "getopt" -Tag 'Scoop' {
     it 'handle short option with required argument missing' {
         $null, $null, $err = getopt '-x' 'x:' ''
         $err | should be 'Option -x requires an argument.'
diff --git a/test/Scoop-Install.Tests.ps1 b/test/Scoop-Install.Tests.ps1
index 8cfebc14e0..a803b555d0 100644
--- a/test/Scoop-Install.Tests.ps1
+++ b/test/Scoop-Install.Tests.ps1
@@ -6,7 +6,7 @@
 
 $isUnix = is_unix
 
-describe "ensure_architecture" {
+describe "ensure_architecture" -Tag 'Scoop' {
     it "should keep correct architectures" {
         ensure_architecture "32bit" | Should be "32bit"
         ensure_architecture "32" | Should be "32bit"
@@ -37,13 +37,13 @@ describe "ensure_architecture" {
     }
 }
 
-describe "appname_from_url" {
+describe "appname_from_url" -Tag 'Scoop' {
     it "should extract the correct name" {
         appname_from_url "https://example.org/directory/foobar.json" | Should be "foobar"
     }
 }
 
-describe "url_filename" {
+describe "url_filename" -Tag 'Scoop' {
     it "should extract the real filename from an url" {
         url_filename "http://example.org/foo.txt" | Should be "foo.txt"
         url_filename "http://example.org/foo.txt?var=123" | Should be "foo.txt"
@@ -54,7 +54,7 @@ describe "url_filename" {
     }
 }
 
-describe "url_remote_filename" {
+describe "url_remote_filename" -Tag 'Scoop' {
     it "should extract the real filename from an url" {
         url_remote_filename "http://example.org/foo.txt" | Should be "foo.txt"
         url_remote_filename "http://example.org/foo.txt?var=123" | Should be "foo.txt"
@@ -65,7 +65,7 @@ describe "url_remote_filename" {
     }
 }
 
-describe "is_in_dir" {
+describe "is_in_dir" -Tag 'Scoop' {
     it "should work correctly" -skip:$isUnix {
         is_in_dir "C:\test" "C:\foo" | Should be $false
         is_in_dir "C:\test" "C:\test\foo\baz.zip" | Should be $true
@@ -75,7 +75,7 @@ describe "is_in_dir" {
     }
 }
 
-describe "env add and remove path" {
+describe "env add and remove path" -Tag 'Scoop' {
     # test data
     $manifest = @{
         "env_add_path" = @("foo", "bar")
@@ -101,7 +101,7 @@ describe "env add and remove path" {
     }
 }
 
-describe "shim_def" {
+describe "shim_def" -Tag 'Scoop' {
     it "should use strings correctly" {
         $target, $name, $shimArgs = shim_def "command.exe"
         $target | Should be "command.exe"
@@ -122,7 +122,7 @@ describe "shim_def" {
     }
 }
 
-describe 'persist_def' {
+describe 'persist_def' -Tag 'Scoop' {
     it 'parses string correctly' {
         $source, $target = persist_def "test"
         $source | Should be "test"
@@ -153,7 +153,7 @@ describe 'persist_def' {
     }
 }
 
-describe 'compute_hash' {
+describe 'compute_hash' -Tag 'Scoop' {
     beforeall {
         $working_dir = setup_working "manifest"
     }
diff --git a/test/Scoop-Linting.Tests.ps1 b/test/Scoop-Linting.Tests.ps1
index b209f93f07..aa3f929460 100644
--- a/test/Scoop-Linting.Tests.ps1
+++ b/test/Scoop-Linting.Tests.ps1
@@ -1,6 +1,6 @@
 $repo_dir = (Get-Item $MyInvocation.MyCommand.Path).directory.parent.FullName
 
-Describe "PSScriptAnalyzer" {
+Describe -Tag 'Linter' "PSScriptAnalyzer" {
     $scoop_modules = Get-ChildItem $repo_dir -Recurse -Include *.psd1, *.psm1, *.ps1
     $scoop_modules = $scoop_modules | Where-Object { $_.DirectoryName -notlike '*\supporting*' -and $_.DirectoryName -notlike '*\test*' }
     $scoop_modules = $scoop_modules | Select-Object -ExpandProperty Directory -Unique
diff --git a/test/Scoop-Manifest.Tests.ps1 b/test/Scoop-Manifest.Tests.ps1
index 9507706704..ade00acc5d 100644
--- a/test/Scoop-Manifest.Tests.ps1
+++ b/test/Scoop-Manifest.Tests.ps1
@@ -2,7 +2,7 @@
 . "$psscriptroot\..\lib\core.ps1"
 . "$psscriptroot\..\lib\manifest.ps1"
 
-describe "manifest-validation" {
+describe -Tag 'Manifests' "manifest-validation" {
     beforeall {
         $working_dir = setup_working "manifest"
         $schema = "$psscriptroot/../schema.json"
@@ -48,6 +48,11 @@ describe "manifest-validation" {
             if ($null -eq $bucketdir) {
                 $bucketdir = "$psscriptroot\..\bucket\"
             }
+            $changed_manifests = @()
+            if($env:CI -eq $true) {
+                $commit = if($env:APPVEYOR_PULL_REQUEST_HEAD_COMMIT) { $env:APPVEYOR_PULL_REQUEST_HEAD_COMMIT } else { $env:APPVEYOR_REPO_COMMIT }
+                $changed_manifests = (Get-GitChangedFile -Include '*.json' -Commit $commit)
+            }
             $manifest_files = Get-ChildItem $bucketdir *.json
             $validator = new-object Scoop.Validator($schema, $true)
         }
@@ -55,7 +60,11 @@ describe "manifest-validation" {
         $quota_exceeded = $false
 
         $manifest_files | ForEach-Object {
-            it "$_" {
+            $skip_manifest = ($changed_manifests -inotcontains $_.FullName)
+            if($env:CI -ne $true -or $changed_manifests -contains 'schema.json') {
+                $skip_manifest = $false
+            }
+            it "$_" -skip:$skip_manifest {
                 $file = $_ # exception handling may overwrite $_
 
                 if(!($quota_exceeded)) {
diff --git a/test/Scoop-Versions.Tests.ps1 b/test/Scoop-Versions.Tests.ps1
index 42a2972495..92ef774e51 100644
--- a/test/Scoop-Versions.Tests.ps1
+++ b/test/Scoop-Versions.Tests.ps1
@@ -1,7 +1,7 @@
 . "$psscriptroot\Scoop-TestLib.ps1"
 . "$psscriptroot\..\lib\versions.ps1"
 
-describe "versions" {
+describe "versions" -Tag 'Scoop' {
     it 'compares versions with integer-string mismatch' {
         $a = '1.8.9'
         $b = '1.8.5-1'
diff --git a/test/bin/init.ps1 b/test/bin/init.ps1
new file mode 100644
index 0000000000..a35dd563ba
--- /dev/null
+++ b/test/bin/init.ps1
@@ -0,0 +1,20 @@
+Write-Host "PowerShell: $($PSVersionTable.PSVersion)"
+
+Write-Host "Install dependencies ..."
+Install-Module -Repository PSGallery -Scope CurrentUser -Force -Name Pester -SkipPublisherCheck
+Install-Module -Repository PSGallery -Scope CurrentUser -Name PSScriptAnalyzer,BuildHelpers
+
+if($env:CI -eq $true) {
+    Write-Host "Load 'BuildHelpers' environment variables ..."
+    Set-BuildEnvironment -Force
+}
+
+$buildVariables = ( Get-ChildItem -Path 'Env:' ).Where( { $_.Name -match "^(?:BH|CI(?:_|$)|APPVEYOR)" } )
+$buildVariables += ( Get-Variable -Name 'CI_*' -Scope 'Script' )
+$details = $buildVariables |
+    Where-Object -FilterScript { $_.Name -notmatch 'EMAIL' } |
+    Sort-Object -Property 'Name' |
+    Format-Table -AutoSize -Property 'Name', 'Value' |
+    Out-String
+Write-Host "CI variables:"
+Write-Host $details -ForegroundColor DarkGray
diff --git a/test/bin/test.ps1 b/test/bin/test.ps1
new file mode 100644
index 0000000000..d593451c43
--- /dev/null
+++ b/test/bin/test.ps1
@@ -0,0 +1,55 @@
+#requires -Version 5.0
+#requires -Modules @{ ModuleName = 'BuildHelpers'; ModuleVersion = '1.2.0' }
+#requires -Modules @{ ModuleName = 'Pester'; ModuleVersion = '4.4.0' }
+#requires -Modules @{ ModuleName = 'PSScriptAnalyzer'; ModuleVersion = '1.17.1' }
+
+$resultsXml = "$PSScriptRoot/TestResults.xml"
+$excludes = @()
+
+$splat = @{
+    Path = 'test/'
+    OutputFile = $resultsXml
+    OutputFormat = 'NUnitXML'
+    PassThru = $true
+}
+
+if($env:CI -eq $true) {
+    $commit = if($env:APPVEYOR_PULL_REQUEST_HEAD_COMMIT) { $env:APPVEYOR_PULL_REQUEST_HEAD_COMMIT } else { $env:APPVEYOR_REPO_COMMIT }
+    $commitMessage = "$env:APPVEYOR_REPO_COMMIT_MESSAGE $env:APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED".TrimEnd()
+
+    if($commitMessage -match '!linter') {
+        Write-Warning "Skipping code linting per commit flag '!linter'"
+        $excludes += 'Linter'
+    }
+
+    $changed_scripts = (Get-GitChangedFile -Include '*.ps1' -Commit $commit)
+    if(!$changed_scripts) {
+        Write-Warning "Skipping tests and code linting for *.ps1 files because they didn't change"
+        $excludes += 'Linter'
+        $excludes += 'Scoop'
+    }
+
+    if($commitMessage -match '!manifests') {
+        Write-Warning "Skipping manifest validation per commit flag '!manifests'"
+        $excludes += 'Manifests'
+    }
+
+    $changed_manifests = (Get-GitChangedFile -Include '*.json' -Commit $commit)
+    if(!$changed_manifests) {
+        Write-Warning "Skipping tests and validation for manifest files because they didn't change"
+        $excludes += 'Manifests'
+    }
+
+    if($excludes.Length -gt 0) {
+        $splat.ExcludeTag = $excludes
+    }
+}
+
+Write-Host 'Invoke-Pester' @splat
+$result = Invoke-Pester @splat
+
+(New-Object Net.WebClient).UploadFile("https://ci.appveyor.com/api/testresults/nunit/$($env:APPVEYOR_JOB_ID)", $resultsXml)
+
+if($result.FailedCount -gt 0) {
+    exit $result.FailedCount
+}