Periodic refresh process for project entries

This will attempt to refresh the project
information, first using the stored
etag to check the latest update on GitHub.

ref: #74

lib/ghtorrent.rb

@@ -53,6 +53,7 @@ module GHTorrent
 require 'ghtorrent/event_processing'
 require 'ghtorrent/ghtorrent'
 require 'ghtorrent/transacted_gh_torrent'
+require 'ghtorrent/refresher'
 
 # Multi-process queue clients
 require 'ghtorrent/multiprocess_queue_client'

lib/ghtorrent/ghtorrent.rb

@@ -6,6 +6,7 @@
 require 'ghtorrent/retriever'
 require 'ghtorrent/persister'
 require 'ghtorrent/geolocator'
+require 'ghtorrent/refresher'
 
 module GHTorrent
   class Mirror
@@ -15,6 +16,7 @@ class Mirror
     include GHTorrent::Retriever
     include GHTorrent::Persister
     include GHTorrent::Geolocator
+    include GHTorrent::Refresher
 
     attr_reader :settings, :persister, :logger
 
@@ -564,7 +566,7 @@ def ensure_repo(user, repo, recursive = false)
 
       unless currepo.nil?
         debug "Repo #{user}/#{repo} exists"
-        return currepo
+        return refresh_repo(user, repo, currepo)
       end
 
       r = retrieve_repo(user, repo, true)
@@ -585,7 +587,8 @@ def ensure_repo(user, repo, recursive = false)
                    :description => unless r['description'].nil? then r['description'][0..254] else nil end,
                    :language => r['language'],
                    :created_at => date(r['created_at']),
-                   :updated_at => Time.at(86400))
+                   :updated_at => Time.now.to_i,
+                   :etag => unless r['etag'].nil? then r['etag'] end)
 
       unless r['parent'].nil?
         parent_owner = r['parent']['owner']['login']

lib/ghtorrent/refresher.rb

@@ -0,0 +1,30 @@
+require 'ghtorrent/api_client'
+require 'ghtorrent/settings'
+require 'ghtorrent/retriever'
+
+module GHTorrent
+  module Refresher
+
+    def refresh_repo(owner, repo, db_entry)
+
+      return db_entry if Time.now.to_i - db_entry[:updated_at].to_i > 3600 * 24
+
+      etag = db_entry[:etag]
+      url = ghurl "repos/#{owner}/#{repo}"
+
+      if last_updated(url, etag).to_i > db_entry[:updated_at].to_i
+        fresh_repo = retrieve_repo(owner, repo, true)
+
+        unless fresh_repo.nil?
+          db.from(:projects).
+              where(:id => db_entry[:id]).
+              update(:etag => fresh_repo['etag'])
+        end
+
+        return db[:projects].first(:id => db_entry[:id])
+      end
+
+      db_entry
+    end
+  end
+end

lib/ghtorrent/retriever.rb

@@ -230,6 +230,7 @@ def retrieve_repo(user, repo, refresh = false)
 
         if refresh
           persister.upsert(:repos, {'name' => r['name'], 'owner.login' => r['owner']['login']}, r)
+          info "Refreshed repo #{user} -> #{repo}"
         else
           persister.store(:repos, r)
           info "Added repo #{user} -> #{repo}"