Merge pull request #108 from kkamkou/issue-103

Issue 103

Readme.md

@@ -25,7 +25,8 @@ py.test --pep8 --clearcache --cov gitmostwanted tests/unit
 ```
 
 ## Docker
+The first time [it'll fail](/~https://github.com/docker/compose/issues/374)
 
 ```bash
-[sudo] docker-compose up -d  # The first time it'll fail
+[sudo] docker-compose up -d
 ```

docker-compose.yml

@@ -1,7 +1,7 @@
 web:
   build: .
   user: gitmostwanted
-  entrypoint: ["/opt/entrypoint.sh"]
+  entrypoint: ["/opt/entrypoint.sh", "web"]
   ports:
     - "5000:5000"
   volumes:
@@ -15,7 +15,7 @@ web:
 celery:
   build: .
   user: gitmostwanted
-  command: celery -A gitmostwanted.app.celery worker -B -s /tmp/celerybeat-schedule.dat --autoreload --loglevel=DEBUG
+  entrypoint: ["/opt/entrypoint.sh", "celery"]
   environment:
       - GMW_APP_SETTINGS=/opt/instance.cfg
   volumes_from:

entrypoint.sh

@@ -1,6 +1,18 @@
 #!/bin/bash
 set -e
 
-(cd ..; alembic upgrade head)
+case "$1" in
+    web)
+        (cd ..; alembic upgrade head)
+        python -m gitmostwanted.web
+        ;;
 
-python -m gitmostwanted.web
+    celery)
+        rm -f /tmp/celerybeat-schedule.dat
+        celery -A gitmostwanted.app.celery worker -B -s /tmp/celerybeat-schedule.dat --autoreload --loglevel=DEBUG
+        ;;
+
+    *)
+        echo $"Usage: $0 {web|celery}"
+        exit 1
+esac

gitmostwanted/blueprints/repo_rating.py

@@ -1,6 +1,5 @@
 from flask import Blueprint, g, request, render_template
 from gitmostwanted.app import db
-from gitmostwanted.blueprints.mixin import repository_filtered
 from gitmostwanted.models.user import UserAttitude
 from gitmostwanted.models.repo import Repo
 
@@ -24,11 +23,21 @@ def top(page):
         .order_by(Repo.worth.desc()) \
         .order_by(Repo.created_at.asc())
 
-    l = Repo.language_distinct()
-    q = repository_filtered(request, l, Repo, q)
+    lngs = Repo.language_distinct()
+
+    lang = request.args.get('lang')
+    if lang != 'All' and (lang,) in lngs:
+        q = q.filter(Repo.language == lang)
+
+    status = request.args.get('status')
+    if status in ('promising', 'hopeless'):
+        q = q.filter(Repo.status == status)
+
+    if bool(request.args.get('mature')):
+        q = q.filter(Repo.mature.is_(True))
 
     entries = q.paginate(page if page > 0 else 1, per_page=20, error_out=False)
     if entries.pages and entries.pages < entries.page:
         return top(entries.pages)
 
-    return render_template('repository/top.html', repos=entries, page=page, languages=l)
+    return render_template('repository/top.html', repos=entries, page=page, languages=lngs)

gitmostwanted/blueprints/repo_trending.py

@@ -0,0 +1,42 @@
+from flask import Blueprint, g, request, render_template
+from gitmostwanted.models.repo import Repo
+from gitmostwanted.models.user import UserAttitude
+from gitmostwanted.models import report
+from gitmostwanted.app import db
+
+repo_trending = Blueprint('repo_trending', __name__)
+
+
+@repo_trending.route('/', defaults={'rng': 'day'})
+@repo_trending.route('/trending/<rng>/')
+def list_by_range(rng):
+    map_list = {'day': 'ReportAllDaily', 'week': 'ReportAllWeekly', 'month': 'ReportAllMonthly'}
+    model = getattr(report, map_list.get(rng, map_list.get('day')))
+
+    query = model.query.join(Repo).order_by(model.cnt_watch.desc())
+    if not g.user:
+        return list_by_range_filtered(query.add_columns(db.null()))
+
+    return list_by_range_filtered(
+        query.add_columns(UserAttitude.attitude).outerjoin(
+            UserAttitude,
+            (UserAttitude.user_id == g.user.id) & (UserAttitude.repo_id == Repo.id)
+        )
+    )
+
+
+def list_by_range_filtered(query):
+    langs = Repo.language_distinct()
+
+    lang = request.args.get('lang')
+    if lang != 'All' and (lang,) in langs:
+        query = query.filter(Repo.language == lang)
+
+    status = request.args.get('status')
+    if status in ('promising', 'hopeless'):
+        query = query.filter(Repo.status == status)
+
+    if bool(request.args.get('mature')):
+        query = query.filter(Repo.mature.is_(True))
+
+    return render_template('index.html', entries=query, languages=langs)

gitmostwanted/blueprints/user_attitude.py

@@ -1,6 +1,5 @@
-from flask import\
-    abort, Blueprint, g, make_response, request, render_template_string, render_template
-from gitmostwanted.blueprints.mixin import repository_filtered
+from flask import abort, Blueprint, g, make_response, request, render_template_string,\
+    render_template
 from gitmostwanted.models.user import UserAttitude
 from gitmostwanted.models.repo import Repo
 from gitmostwanted.app import db
@@ -56,7 +55,16 @@ def list_by_attitude(attitude, page):
 
     languages = Repo.language_distinct()
 
-    q = repository_filtered(request, languages, Repo, q)
+    lang = request.args.get('lang')
+    if lang != 'All' and (lang,) in languages:
+        q = q.filter(Repo.language == lang)
+
+    status = request.args.get('status')
+    if status in ('promising', 'hopeless'):
+        q = q.filter(Repo.status == status)
+
+    if bool(request.args.get('mature')):
+        q = q.filter(Repo.mature.is_(True))
 
     entries = q.paginate(page if page > 0 else 1, per_page=20, error_out=False)
     if entries.pages and entries.pages < entries.page:

gitmostwanted/blueprints/user_oauth.py

@@ -0,0 +1,62 @@
+from flask import Blueprint, g, redirect, request, session, url_for
+from gitmostwanted.services import oauth as service_oauth
+from gitmostwanted.models.user import User
+from gitmostwanted.app import app, db
+
+user_oauth = Blueprint('user_oauth', __name__)
+oauth = service_oauth.instance(app)
+
+
+@app.before_request
+def load_user_from_session():
+    if str(request.url_rule) in ['/logout']:
+        return None
+    g.user = User.query.get(session['user_id']) if 'user_id' in session else None
+
+
+@user_oauth.route('/logout')
+def logout():
+    session.pop('github_token', None)
+    session.pop('user_id', None)
+    return redirect('/')
+
+
+@user_oauth.route('/oauth/login')
+def login():
+    return oauth.github\
+        .authorize(callback=url_for('user_oauth.authorized', next=url_next(), _external=True))
+
+
+@user_oauth.route('/oauth/authorized')
+def authorized():
+    next_url = url_next() or url_for('/')
+
+    resp = oauth.github.authorized_response()
+    if resp is None:
+        return redirect(next_url)
+
+    session.permanent = True
+    session['github_token'] = (resp['access_token'], '')
+    me = oauth.github.get('user')
+    session['user_id'] = user_get_or_create(me.data['id'], me.data['email'], me.data['login']).id
+
+    return redirect(next_url)
+
+
+@oauth.github.tokengetter
+def github_tokengetter():
+    return session.get('github_token')
+
+
+def user_get_or_create(uid, uemail, uname):
+    entity = User.query.filter_by(github_id=uid).first()
+    if entity:
+        return entity
+    entity = User(github_id=uid, username=uname, email=uemail or None)
+    db.session.add(entity)
+    db.session.commit()
+    return entity
+
+
+def url_next():
+    return request.args.get('next') or request.referrer or None

gitmostwanted/bigquery/job.py → gitmostwanted/lib/bigquery/job.py

@@ -1,5 +1,5 @@
-from gitmostwanted.bigquery.service import ServiceGmw
-from gitmostwanted.bigquery.result import ResultJob
+from gitmostwanted.lib.bigquery.service import ServiceGmw
+from gitmostwanted.lib.bigquery.result import ResultJob
 from itertools import chain
 from uuid import uuid4
 
@@ -9,9 +9,27 @@ class Job:
     num_retries = 5
 
     def __init__(self, api: ServiceGmw, query_str: str, batch: bool=False):
-        self.__complete = False
+        self.__complete = self.__id = False
+        self.__query_str = query_str
+        self.__batch = batch
         self.__api = api
-        self.__id = self.__insert(query_str, batch)
+
+    def __insert(self, query_str: str, batch: bool=False):
+        body = {
+            'jobReference': {
+                'projectId': self.__api.project_id,
+                'job_id': str(uuid4())
+            },
+            'configuration': {
+                'query': {
+                    'query': query_str,
+                    'priority': 'BATCH' if batch else 'INTERACTIVE'
+                }
+            }
+        }
+        return self.__api.jobs()\
+            .insert(projectId=self.__api.project_id, body=body)\
+            .execute(num_retries=self.num_retries)['jobReference']['jobId']
 
     @property
     def id(self):
@@ -25,6 +43,8 @@ def complete(self):
 
     @property
     def info(self):
+        if not self.id:
+            raise JobNotStartedException('The job is not started yet')
         return self.__api.jobs()\
             .get(projectId=self.__api.project_id, jobId=self.id)\
             .execute(num_retries=self.num_retries)
@@ -51,23 +71,14 @@ def pages(self):
 
             yield ResultJob(response)
 
-    def __insert(self, query_str, batch: bool=False):
-        body = {
-            'jobReference': {
-                'projectId': self.__api.project_id,
-                'job_id': str(uuid4())
-            },
-            'configuration': {
-                'query': {
-                    'query': query_str,
-                    'priority': 'BATCH' if batch else 'INTERACTIVE'
-                }
-            }
-        }
-        return self.__api.jobs()\
-            .insert(projectId=self.__api.project_id, body=body)\
-            .execute(num_retries=self.num_retries)['jobReference']['jobId']
+    def execute(self):
+        self.__id = self.__insert(self.__query_str, self.__batch)
+        return self.id
 
 
 class JobIncompleteException(Exception):
     pass
+
+
+class JobNotStartedException(Exception):
+    pass

gitmostwanted/services/bigquery.py

@@ -1,4 +1,4 @@
-from gitmostwanted.bigquery.service import ServiceGmw
+from gitmostwanted.lib.bigquery.service import ServiceGmw
 
 
 def instance(app):

gitmostwanted/tasks/repo_metadata.py

@@ -1,6 +1,6 @@
 from gitmostwanted.app import app, db, celery
 from gitmostwanted.models.repo import Repo
-from gitmostwanted.github import api
+from gitmostwanted.lib.github import api
 from datetime import datetime, timedelta
 
 

gitmostwanted/tasks/repo_most_starred.py

@@ -1,14 +1,14 @@
-from gitmostwanted.github.api import repo_info
+from gitmostwanted.lib.github.api import repo_info
+from gitmostwanted.lib.bigquery.job import Job
 from gitmostwanted.app import app, db, celery
 from gitmostwanted.services import bigquery
-from gitmostwanted.bigquery.job import Job
 from gitmostwanted.models.repo import Repo
 from gitmostwanted.models import report
 from datetime import date, datetime
 from time import sleep
 
 
-def job_results(j: Job):
+def results_of(j: Job):
     while not j.complete:
         app.logger.debug('The job is not complete, waiting...')
         sleep(10)
@@ -76,12 +76,15 @@ def most_starred_month():
 def most_starred_sync(model_name: str, query: str):
     app.logger.info('Importing repos of %s', model_name)
 
-    service = bigquery.instance(app)
     model = getattr(report, model_name)
+    service = bigquery.instance(app)
 
     db.session.query(model).delete()
 
-    for row in job_results(Job(service, query)):
+    job = Job(service, query)
+    job.execute()
+
+    for row in results_of(job):
         info, code = repo_info(row[1])
         if not info:
             continue

gitmostwanted/tasks/repo_stars.py

@@ -1,12 +1,12 @@
 from gitmostwanted.models.repo import Repo, RepoStars
 from gitmostwanted.app import app, db, celery
 from gitmostwanted.services import bigquery
-from gitmostwanted.bigquery.job import Job
+from gitmostwanted.lib.bigquery.job import Job
 from datetime import datetime, timedelta
 from time import sleep
 
 
-def job_results(j: Job):
+def results_of(j: Job):
     while not j.complete:
         app.logger.debug('The job is not complete, waiting...')
         sleep(10)
@@ -31,10 +31,12 @@ def stars_mature(num_days):
 
     repos = Repo.query.filter(Repo.mature.is_(True)).filter(Repo.status == 'new')
     for repo in repos:
-        jobs.append((Job(service, query.format(id=repo.id, date_from=date_from), batch=True), repo))
+        job = Job(service, query.format(id=repo.id, date_from=date_from), batch=True)
+        job.execute()
+        jobs.append((job, repo))
 
     for job in jobs:
-        for row in job_results(job[0]):
+        for row in results_of(job[0]):
             db.session.add(RepoStars(repo_id=job[1].id, stars=row[0], year=row[1], day=row[2]))
 
         job[1].status = 'unknown'