diff --git a/storage/filestore.go b/storage/filestore.go
index b19ed8aca..67f160b03 100644
--- a/storage/filestore.go
+++ b/storage/filestore.go
@@ -67,52 +67,66 @@ type FilesystemStore struct {
 	perms   os.FileMode
 }
 
+func (f *FilesystemStore) moveKeyTo0Dot4Location(file string) {
+	keyID := filepath.Base(file)
+	fileDir := filepath.Dir(file)
+	d, _ := f.Get(file)
+	block, _ := pem.Decode(d)
+	if block == nil {
+		logrus.Warn("Key data for", file, "could not be decoded as a valid PEM block. The key will not been migrated and may not be available")
+		return
+	}
+	fileDir = strings.TrimPrefix(fileDir, notary.RootKeysSubdir)
+	fileDir = strings.TrimPrefix(fileDir, notary.NonRootKeysSubdir)
+	if fileDir != "" {
+		block.Headers["gun"] = fileDir[1:]
+	}
+	if strings.Contains(keyID, "_") {
+		role := strings.Split(keyID, "_")[1]
+		keyID = strings.TrimSuffix(keyID, "_"+role)
+		block.Headers["role"] = role
+	}
+	var keyPEM bytes.Buffer
+	// since block came from decoding the PEM bytes in the first place, and all we're doing is adding some headers we ignore the possibility of an error while encoding the block
+	pem.Encode(&keyPEM, block)
+	f.Set(keyID, keyPEM.Bytes())
+}
+
 func (f *FilesystemStore) migrateTo0Dot4() {
-	for _, file := range f.ListFiles() {
-		keyID := filepath.Base(file)
-		fileDir := filepath.Dir(file)
-		d, _ := f.Get(file)
-		block, _ := pem.Decode(d)
-		if block == nil {
-			logrus.Warn("Key data for ", file, " may have been tampered with/ is invalid. The key has not been migrated and may not be available")
-			continue
-		}
-		if strings.HasPrefix(fileDir, notary.RootKeysSubdir) {
-			fileDir = strings.TrimPrefix(fileDir, notary.RootKeysSubdir)
-			if strings.Contains(keyID, "_") {
-				role := strings.Split(keyID, "_")[1]
-				keyID = strings.TrimSuffix(keyID, "_"+role)
-				block.Headers["role"] = role
+	rootKeysSubDir := filepath.Clean(filepath.Join(f.Location(), notary.RootKeysSubdir))
+	nonRootKeysSubDir := filepath.Clean(filepath.Join(f.Location(), notary.NonRootKeysSubdir))
+	if _, err := os.Stat(rootKeysSubDir); !os.IsNotExist(err) && f.Location() != rootKeysSubDir {
+		if rootKeysSubDir == "" || rootKeysSubDir == "/" {
+			// making sure we don't remove a user's homedir
+			logrus.Warn("The directory for root keys is an unsafe value, we are not going to delete the directory. Please delete it manually")
+		} else {
+			// root_keys exists, migrate things from it
+			listOnlyRootKeysDirStore, _ := NewFileStore(rootKeysSubDir, f.ext, notary.PrivKeyPerms)
+			for _, file := range listOnlyRootKeysDirStore.ListFiles() {
+				f.moveKeyTo0Dot4Location(filepath.Join(notary.RootKeysSubdir, file))
 			}
-		} else if strings.HasPrefix(fileDir, notary.NonRootKeysSubdir) {
-			fileDir = strings.TrimPrefix(fileDir, notary.NonRootKeysSubdir)
-			if fileDir != "" {
-				block.Headers["gun"] = fileDir[1:]
-			}
-			if strings.Contains(keyID, "_") {
-				role := strings.Split(keyID, "_")[1]
-				keyID = strings.TrimSuffix(keyID, "_"+role)
-				block.Headers["role"] = role
+			// delete the old directory
+			os.RemoveAll(rootKeysSubDir)
+		}
+	}
+
+	if _, err := os.Stat(nonRootKeysSubDir); !os.IsNotExist(err) && f.Location() != nonRootKeysSubDir {
+		if nonRootKeysSubDir == "" || nonRootKeysSubDir == "/" {
+			// making sure we don't remove a user's homedir
+			logrus.Warn("The directory for non root keys is an unsafe value, we are not going to delete the directory. Please delete it manually")
+		} else {
+			// tuf_keys exists, migrate things from it
+			listOnlyNonRootKeysDirStore, _ := NewFileStore(nonRootKeysSubDir, f.ext, notary.PrivKeyPerms)
+			for _, file := range listOnlyNonRootKeysDirStore.ListFiles() {
+				f.moveKeyTo0Dot4Location(filepath.Join(notary.NonRootKeysSubdir, file))
 			}
+			// delete the old directory
+			os.RemoveAll(nonRootKeysSubDir)
 		}
-		var keyPEM bytes.Buffer
-		// since block came from decoding the PEM bytes in the first place, and all we're doing is adding some headers we ignore the possibility of an error while encoding the block
-		pem.Encode(&keyPEM, block)
-		f.Set(keyID, keyPEM.Bytes())
 	}
-	rootKeysSubDir := filepath.Join(f.Location(), notary.RootKeysSubdir)
-	nonRootKeysSubDir := filepath.Join(f.Location(), notary.NonRootKeysSubdir)
+
+	// if we have a trusted_certificates folder, let's delete for a complete migration since it is unused by new clients
 	certsSubDir := filepath.Join(f.Location(), "trusted_certificates")
-	if rootKeysSubDir == "" || rootKeysSubDir == "/" {
-		logrus.Warn("The directory for root keys is an unsafe value, we are not going to delete the directory. Please delete it manually")
-	} else {
-		os.RemoveAll(rootKeysSubDir)
-	}
-	if nonRootKeysSubDir == "" || nonRootKeysSubDir == "/" {
-		logrus.Warn("The directory for non root keys is an unsafe value, we are not going to delete the directory. Please delete it manually")
-	} else {
-		os.RemoveAll(nonRootKeysSubDir)
-	}
 	if certsSubDir == "" || certsSubDir == "/" {
 		logrus.Warn("The directory for trusted certificate is an unsafe value, we are not going to delete the directory. Please delete it manually")
 	} else {
diff --git a/trustmanager/keystore.go b/trustmanager/keystore.go
index a2c2f95d8..bc12e7afd 100644
--- a/trustmanager/keystore.go
+++ b/trustmanager/keystore.go
@@ -40,8 +40,7 @@ func NewKeyFileStore(baseDir string, p notary.PassRetriever) (*GenericKeyStore,
 	if err != nil {
 		return nil, err
 	}
-	store := NewGenericKeyStore(fileStore, p)
-	return store, nil
+	return NewGenericKeyStore(fileStore, p), nil
 }
 
 // NewKeyMemoryStore returns a new KeyMemoryStore which holds keys in memory