From 42448c865366c505e178ac6e646b9cdbc0c6e2fe Mon Sep 17 00:00:00 2001 From: Leonard Schuetz Date: Mon, 6 Jun 2016 11:35:46 +0200 Subject: [PATCH 1/9] Fix draft for #9 --- test.js | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 test.js diff --git a/test.js b/test.js new file mode 100644 index 0000000..151048e --- /dev/null +++ b/test.js @@ -0,0 +1,44 @@ +const gh = require('./index'); +gh.scrapeContributionDataAndStats('/~https://github.com/KCreate', (data) => { + + let longestStreak = 0; + let lastDay = '0-0-0'; + + // Reduce the total number of contributions to a single integer + const totalContributions = data.contributionData.reduce((last, current, index) => { + + // Calculate the exptected day to continue the streak + let expectedDay = new Date(lastDay); + expectedDay.setDate(expectedDay.getDate() + 1); // Increment day by 1 + + const year = expectedDay.getFullYear(); + let month = expectedDay.getMonth() + 1; // The month starts from zero + let day = expectedDay.getDate(); + + // Left-pad for noobs + if (month < 10) { month = '0' + month; } + if (day < 10) { day = '0' + day; } + + expectedDay = ( + year + '-' + + month + '-' + + day + ); + console.log(expectedDay, current.dataDate); + + // If the streak was continued, increment, else reset + if (expectedDay === current.dataDate) { + longestStreak++; + } else { + longestStreak = 0; + } + + // Update the last day + lastDay = current.dataDate; + + return last + current.dataContributionCount; + }, 0); + + console.log('total contributions: ' + totalContributions); + console.log('longest streak: ' + longestStreak); +}); From f7764ab560ae59cde67d20bebc0d4e0fcce8b865 Mon Sep 17 00:00:00 2001 From: Leonard Schuetz Date: Mon, 6 Jun 2016 13:14:04 +0200 Subject: [PATCH 2/9] Fixed scrapeContributionStatsAndData method and added test --- index.js | 88 ++++++++++++++++++++++++++++++++++++++++---------------- test.js | 46 ++++------------------------- 2 files changed, 69 insertions(+), 65 deletions(-) diff --git a/index.js b/index.js index 06dfea7..4edef81 100644 --- a/index.js +++ b/index.js @@ -43,31 +43,71 @@ exports.scrapeContributionStats = function (url, callback) { exports.scrapeContributionDataAndStats = function(url, callback) { var returnObj = {}; getRequest(url, function(html) { - scrapeContributionStats(html, function(statsData) { - // TODO implement better error handling - if (statsData == null) { - getRequest(url, function(html) { - scrapeContributionStats(html, function(statsData) { - scrapeContributionData(html, function(contributionData) { - returnObj = formatReturnData(contributionData, statsData); - callback(returnObj); - }); - }); - }); - } else { - scrapeContributionData(html, function(contributionData) { - returnObj = formatReturnData(contributionData, statsData); - callback(returnObj); - }); - } + scrapeContributionData(html, function(contributionData) { + returnObj = formatReturnData(contributionData, deriveContributionStats(contributionData)); + callback(returnObj); }); }); }; // Helper functions +function deriveContributionStats(contributionData) { + + // Some variables + var longestStreak = 0; + var currentStreak = 0; + var lastDay = '0-0-0'; + + // Reduce the total number of contributions to a single integer + const totalContributions = contributionData.reduce(function (last, current, index) { + + // Calculate the exptected day to continue the streak + var expectedDay = new Date(lastDay); + expectedDay.setDate(expectedDay.getDate() + 1); // Increment day by 1 + + const year = expectedDay.getFullYear(); + var month = expectedDay.getMonth() + 1; // The month starts from zero + var day = expectedDay.getDate(); + + // Left-pad for noobs + if (month < 10) { month = '0' + month; } + if (day < 10) { day = '0' + day; } + + expectedDay = ( + year + '-' + + month + '-' + + day + ); + + // If the streak was continued, increment, else reset + if (expectedDay === current.dataDate) { + currentStreak++; + } else { + currentStreak = 0; + } + + if (currentStreak > longestStreak) { + longestStreak = currentStreak; + } + + // Update the last day + lastDay = current.dataDate; + + return last + current.dataContributionCount; + }, 0); + + return { + totalContributions: totalContributions, + longestStreak: longestStreak, + currentStreak: currentStreak + }; +} + +// Helper functions + function formatReturnData(contributionData, statsData) { - var commitsToday = getCommitsToday(contributionData); + var commitsToday = getCommitsToday(contributionData); var returnData = {contributionData: contributionData, statsData: statsData, commitsToday: commitsToday}; return commitsToday == null ? null : returnData; }; @@ -119,7 +159,7 @@ function scrapeContributionStats(html, callback) { // If it fails validation return null callback(null); } -}; +}; // Returns an Object Containing Contribution Data // CommitDataObj = { @@ -141,7 +181,7 @@ function scrapeContributionData(html, callback) { commitDataObj.dataContributionCount = dataContributionCount; commitDataObj.dataDate = dataDate; commitDataArray.push(commitDataObj); - } + } }); // Validate it contains data before sending @@ -151,9 +191,9 @@ function scrapeContributionData(html, callback) { // If it fails validation return null callback(null); } -}; +}; -// GET Request Helper Function +// GET Request Helper Function function getRequest(gitUrl, callback) { var options = { url: gitUrl @@ -212,7 +252,7 @@ getRequest("http://www.github.com/skeswa", function(html) { getRequest("http://www.github.com/skeswa", function(html) { scrapeContributionStats(html, function(statsData) { scrapeContributionData(html, function(contributionData) { - var returnObj = formatReturnData(contributionData, statsData); + var returnObj = formatReturnData(contributionData, statsData); console.log(returnObj); }); }); @@ -222,7 +262,7 @@ getRequest("http://www.github.com/skeswa", function(html) { console.log(statsData); scrapeContributionData(html, function(contributionData) { console.log(contributionData); - var returnObj = formatReturnData(contributionData, statsData); + var returnObj = formatReturnData(contributionData, statsData); console.log(returnObj); }); } diff --git a/test.js b/test.js index 151048e..994dc95 100644 --- a/test.js +++ b/test.js @@ -1,44 +1,8 @@ const gh = require('./index'); gh.scrapeContributionDataAndStats('/~https://github.com/KCreate', (data) => { - - let longestStreak = 0; - let lastDay = '0-0-0'; - - // Reduce the total number of contributions to a single integer - const totalContributions = data.contributionData.reduce((last, current, index) => { - - // Calculate the exptected day to continue the streak - let expectedDay = new Date(lastDay); - expectedDay.setDate(expectedDay.getDate() + 1); // Increment day by 1 - - const year = expectedDay.getFullYear(); - let month = expectedDay.getMonth() + 1; // The month starts from zero - let day = expectedDay.getDate(); - - // Left-pad for noobs - if (month < 10) { month = '0' + month; } - if (day < 10) { day = '0' + day; } - - expectedDay = ( - year + '-' + - month + '-' + - day - ); - console.log(expectedDay, current.dataDate); - - // If the streak was continued, increment, else reset - if (expectedDay === current.dataDate) { - longestStreak++; - } else { - longestStreak = 0; - } - - // Update the last day - lastDay = current.dataDate; - - return last + current.dataContributionCount; - }, 0); - - console.log('total contributions: ' + totalContributions); - console.log('longest streak: ' + longestStreak); + if (Object.keys(data.statsData).length > 0) { + console.log('[ OK ] Test passed!'); + } else { + console.log('[ FAIL ] Test failed!'); + } }); From 191896a819c779d6d2e9d62867f2f7911f0b341f Mon Sep 17 00:00:00 2001 From: Leonard Schuetz Date: Mon, 6 Jun 2016 13:33:36 +0200 Subject: [PATCH 3/9] Deprecated functions and added tests --- index.js | 140 +++---------------------------------------------------- test.js | 18 +++++-- 2 files changed, 22 insertions(+), 136 deletions(-) diff --git a/index.js b/index.js index 4edef81..7094592 100644 --- a/index.js +++ b/index.js @@ -3,42 +3,6 @@ var request = require('request'), cheerio = require('cheerio'), moment = require('moment'); -// Export Scrape Contribution Data Function -exports.scrapeContributionData = function (url, callback) { - getRequest(url, function(html) { - scrapeContributionData(html, function(contributionData){ - // TODO implement better error handling - if ( results == null) { - getRequest(url, function(html) { - scrapeContributionData(data, function(contributionData) { - callback(contributionData); - }); - }); - } else { - callback(contributionData); - } - }); - }); -}; - -// Export Scrape Profile Stats Data Function -exports.scrapeContributionStats = function (url, callback) { - getRequest(url, function(html) { - scrapeContributionStats(html, function(statsData) { - // TODO implement better error handling - if (results == null) { - getRequest(url, function(html) { - scrapeContributionStats(html, function(statsData) { - callback(statsData); - }); - }); - } else { - callback(statsData); - } - }); - }); -}; - // Export Scrape Profile Data and Stats function exports.scrapeContributionDataAndStats = function(url, callback) { var returnObj = {}; @@ -51,7 +15,6 @@ exports.scrapeContributionDataAndStats = function(url, callback) { }; // Helper functions - function deriveContributionStats(contributionData) { // Some variables @@ -104,8 +67,6 @@ function deriveContributionStats(contributionData) { }; } -// Helper functions - function formatReturnData(contributionData, statsData) { var commitsToday = getCommitsToday(contributionData); var returnData = {contributionData: contributionData, statsData: statsData, commitsToday: commitsToday}; @@ -130,37 +91,6 @@ function getCommitsToday(contributionData) { return 0; }; -// Returns an Object Containing Contribution Stats -// statDataObj = { -// totalContributions: #, -// longestStreak: #, -// currentStreak: # -// }; -function scrapeContributionStats(html, callback) { - $ = cheerio.load(html); - var statDataObj = {}; - - $('.contrib-number').each(function(index, statData) { - var statData = statData.children[0].data; - - if (index == 0) { - statDataObj.totalContributions = parseInt(statData); - } else if (index === 1) { - statDataObj.longestStreak = parseInt(statData); - } else { - statDataObj.currentStreak = parseInt(statData); - } - }); - - // Validate it contains data before sending - if (statDataObj) { - callback(statDataObj); - } else { - // If it fails validation return null - callback(null); - } -}; - // Returns an Object Containing Contribution Data // CommitDataObj = { // dataContributionCount: #, @@ -206,66 +136,10 @@ function getRequest(gitUrl, callback) { }); }; -// Tests -/* -getRequest("/~https://github.com/shikkic", function(html) { - console.log("Scraping Contribution Data"); - scrapeContributionData(html, function(results) { - if (results == null) { - console.log("result is equal null ", results); - getRequest("/~https://github.com/shikkic", function(html) { - scrapeContributionData(html, function(results) { - console.log(results); - }); - }); - } else { - console.log(results); - } - }); -}); - -getRequest("/~https://github.com/shikkic", function(html) { - console.log("Scraping Contribution Stats"); - scrapeContributionStats(html, function(results) { - if (results == null) { - console.log("result is equal null ", results); - getRequest("/~https://github.com/shikkic", function(html) { - scrapeContributionData(html, function(results) { - console.log(results); - }); - }); - } else { - console.log(results); - } - }); -});*/ - -// Test for scraping both contribution stats and data -/* -getRequest("http://www.github.com/skeswa", function(html) { - //console.log(html); - scrapeContributionStats(html, function(statsData) { - // TODO implement better error handling - console.log(statsData); - if (statsData) { - console.log("Stat data is null"); - getRequest("http://www.github.com/skeswa", function(html) { - scrapeContributionStats(html, function(statsData) { - scrapeContributionData(html, function(contributionData) { - var returnObj = formatReturnData(contributionData, statsData); - console.log(returnObj); - }); - }); - }); - } else { - console.log("Stat data is not null"); - console.log(statsData); - scrapeContributionData(html, function(contributionData) { - console.log(contributionData); - var returnObj = formatReturnData(contributionData, statsData); - console.log(returnObj); - }); - } - }); -}); -*/ +// Deprecated warnings +exports.scrapeContributionData = function() { + console.log('Deprecation warning: scrapeContributionData has been deprecated in favor of just using scrapeContributionDataAndStats.'); +}; +exports.scrapeContributionStats = function() { + console.log('Deprecation warning: scrapeContributionData has been deprecated in favor of just using scrapeContributionDataAndStats.'); +}; diff --git a/test.js b/test.js index 994dc95..9f928fa 100644 --- a/test.js +++ b/test.js @@ -1,8 +1,20 @@ const gh = require('./index'); -gh.scrapeContributionDataAndStats('/~https://github.com/KCreate', (data) => { +const url = '/~https://github.com/KCreate'; + +gh.scrapeContributionDataAndStats(url, (data) => { if (Object.keys(data.statsData).length > 0) { - console.log('[ OK ] Test passed!'); + console.log('[ OK ] Test 1 passed!'); } else { - console.log('[ FAIL ] Test failed!'); + console.log('[ FAIL ] Test 1 failed!'); } }); + + +// These just fail if they are being run. +// The implementation is supposed to show a deprecation warning +gh.scrapeContributionData(url, (data) => { + console.log('[ FAIL ] Test 2 failed'); +}); +gh.scrapeContributionStats(url, (data) => { + console.log('[ FAIL ] Test 3 failed'); +}); From 1b23bb5f3ab9cd948eb97d82f4a3603264ba6797 Mon Sep 17 00:00:00 2001 From: Leonard Schuetz Date: Mon, 6 Jun 2016 13:34:22 +0200 Subject: [PATCH 4/9] Removed underscore as a dependency --- index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/index.js b/index.js index 7094592..890906a 100644 --- a/index.js +++ b/index.js @@ -1,5 +1,4 @@ var request = require('request'), - _ = require('underscore'), cheerio = require('cheerio'), moment = require('moment'); @@ -79,7 +78,7 @@ function getCommitsToday(contributionData) { } // Grab lastest commit data - var latestCommits = _.last(contributionData), + var latestCommits = contributionData[contributionData.length - 1], latestCommitsDate = moment(latestCommits.dataDate).dayOfYear(), todaysDate = moment().dayOfYear(); From 1e5e8cd46b4581238592884babfe083e0a6c6213 Mon Sep 17 00:00:00 2001 From: Leonard Schuetz Date: Mon, 6 Jun 2016 13:42:01 +0200 Subject: [PATCH 5/9] Improved docs --- index.js | 49 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index 890906a..fbdf20a 100644 --- a/index.js +++ b/index.js @@ -2,7 +2,13 @@ var request = require('request'), cheerio = require('cheerio'), moment = require('moment'); -// Export Scrape Profile Data and Stats function + +/** + * List of contributons, latest commit, streak etc. + * + * @param {string} url - URL to the github repo + * @param {function} callback - Receives the data + */ exports.scrapeContributionDataAndStats = function(url, callback) { var returnObj = {}; getRequest(url, function(html) { @@ -13,7 +19,13 @@ exports.scrapeContributionDataAndStats = function(url, callback) { }); }; -// Helper functions + +/** + * Derive the streak + * + * @param {object} contributionData - Contribution data + * @return {object} - Stats derived from the contribution data + */ function deriveContributionStats(contributionData) { // Some variables @@ -66,12 +78,27 @@ function deriveContributionStats(contributionData) { }; } + +/** + * Neatly formats contributionData and statsData + * + * @param {object} contributionData + * @param {object} statsData + * @return {object} - Returns null if no commits have happened today + */ function formatReturnData(contributionData, statsData) { var commitsToday = getCommitsToday(contributionData); var returnData = {contributionData: contributionData, statsData: statsData, commitsToday: commitsToday}; return commitsToday == null ? null : returnData; }; + +/** + * Returns the amount of contributions that happened on the current day + * + * @param {object} contributionData + * @return {integer} - Amount of contributions in the current day + */ function getCommitsToday(contributionData) { if (!contributionData) { return null; @@ -95,6 +122,14 @@ function getCommitsToday(contributionData) { // dataContributionCount: #, // dataDate: #, // }; + + +/** + * Returns an object containing contribution data + * + * @param {string} html - HTML of the github page + * @param {function} callback - Receives contribution data + */ function scrapeContributionData(html, callback) { $ = cheerio.load(html); var commitDataArray = []; @@ -122,12 +157,18 @@ function scrapeContributionData(html, callback) { } }; -// GET Request Helper Function + +/** + * Opiniated get request + * + * @param {string} gitUrl + * @param {function} callback3 + */ function getRequest(gitUrl, callback) { var options = { url: gitUrl }; - request.get(options, function(error, response, body){ + request.get(options, function(error, response, body) { if(!error) { callback(body); } From fce9fe994eebaf9be3b04783f3a42ff5bbc3c2fc Mon Sep 17 00:00:00 2001 From: Leonard Schuetz Date: Mon, 6 Jun 2016 15:29:46 +0200 Subject: [PATCH 6/9] Removed moment and underscore as a dependencie --- index.js | 25 +++++++++++++++++++++---- package.json | 8 +++----- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/index.js b/index.js index fbdf20a..06a8287 100644 --- a/index.js +++ b/index.js @@ -104,10 +104,10 @@ function getCommitsToday(contributionData) { return null; } - // Grab lastest commit data - var latestCommits = contributionData[contributionData.length - 1], - latestCommitsDate = moment(latestCommits.dataDate).dayOfYear(), - todaysDate = moment().dayOfYear(); + // Grab latest commit data + var latestCommits = contributionData[contributionData.length - 1]; + var latestCommitsDate = dayOfYear(latestCommits.dataDate); + var todaysDate = dayOfYear(new Date()); // Check if the lastest commit data is from today or not if (latestCommitsDate === todaysDate) { @@ -176,6 +176,23 @@ function getRequest(gitUrl, callback) { }); }; + +/** + * Returns the day of the year + * + * @param {string, date} date - Can be a string or a timestamp + * @return {integer} The day of the year + */ +function dayOfYear(date) { + var now = new Date(date); + var start = new Date(now.getFullYear(), 0, 0); + var diff = now - start; + var oneDay = 1000 * 60 * 60 * 24; + var day = Math.floor(diff / oneDay); + + return day; +} + // Deprecated warnings exports.scrapeContributionData = function() { console.log('Deprecation warning: scrapeContributionData has been deprecated in favor of just using scrapeContributionDataAndStats.'); diff --git a/package.json b/package.json index 806252f..d1533d9 100644 --- a/package.json +++ b/package.json @@ -1,17 +1,15 @@ { "name": "gh-scrape", - "version": "1.0.10", + "version": "1.0.2", "description": "", "main": "index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "node test" }, "author": "Shikkic", "license": "ISC", "dependencies": { "cheerio": "^0.19.0", - "moment": "^2.11.2", - "request": "^2.64.0", - "underscore": "^1.8.3" + "request": "^2.64.0" } } From 942425413aefc8be3ad17e15a1658778682ad5f7 Mon Sep 17 00:00:00 2001 From: Leonard Schuetz Date: Mon, 6 Jun 2016 19:49:18 +0200 Subject: [PATCH 7/9] Fixed longestStreak and currentStreak count --- index.js | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/index.js b/index.js index 06a8287..4b90cd4 100644 --- a/index.js +++ b/index.js @@ -29,8 +29,8 @@ exports.scrapeContributionDataAndStats = function(url, callback) { function deriveContributionStats(contributionData) { // Some variables - var longestStreak = 0; - var currentStreak = 0; + var longestStreak = 1; + var currentStreak = 1; var lastDay = '0-0-0'; // Reduce the total number of contributions to a single integer @@ -117,13 +117,6 @@ function getCommitsToday(contributionData) { return 0; }; -// Returns an Object Containing Contribution Data -// CommitDataObj = { -// dataContributionCount: #, -// dataDate: #, -// }; - - /** * Returns an object containing contribution data * From 4946778281ea5cfffd9f96d510dd913ee1221bfc Mon Sep 17 00:00:00 2001 From: Leonard Schuetz Date: Mon, 6 Jun 2016 19:53:38 +0200 Subject: [PATCH 8/9] Removed dead require --- index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/index.js b/index.js index 4b90cd4..2584054 100644 --- a/index.js +++ b/index.js @@ -1,6 +1,5 @@ var request = require('request'), - cheerio = require('cheerio'), - moment = require('moment'); + cheerio = require('cheerio'); /** From 8244f6b87e3d0c43afacb3bf861630ad59447a02 Mon Sep 17 00:00:00 2001 From: Leonard Schuetz Date: Mon, 6 Jun 2016 20:06:06 +0200 Subject: [PATCH 9/9] Fixed #9424254 --- index.js | 8 ++++---- test.js | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/index.js b/index.js index 2584054..75cd9ac 100644 --- a/index.js +++ b/index.js @@ -28,8 +28,8 @@ exports.scrapeContributionDataAndStats = function(url, callback) { function deriveContributionStats(contributionData) { // Some variables - var longestStreak = 1; - var currentStreak = 1; + var longestStreak = 0; + var currentStreak = 0; var lastDay = '0-0-0'; // Reduce the total number of contributions to a single integer @@ -72,8 +72,8 @@ function deriveContributionStats(contributionData) { return { totalContributions: totalContributions, - longestStreak: longestStreak, - currentStreak: currentStreak + longestStreak: longestStreak + 1, + currentStreak: currentStreak + 1 }; } diff --git a/test.js b/test.js index 9f928fa..4a43c2f 100644 --- a/test.js +++ b/test.js @@ -7,6 +7,8 @@ gh.scrapeContributionDataAndStats(url, (data) => { } else { console.log('[ FAIL ] Test 1 failed!'); } + + console.log(data); });