From c11ad6c5c3e649e029a272a295e7bd0322025a1c Mon Sep 17 00:00:00 2001 From: Jan Vansteenlandt Date: Mon, 25 Jan 2016 11:04:12 +0100 Subject: [PATCH 1/5] Handle empty rest parameters and update framework to 4.2.1 --- app/Tdt/Core/Datasets/DatasetController.php | 26 ++++----- app/Tdt/Core/Formatters/GEOJSONFormatter.php | 1 + composer.json | 5 +- composer.lock | 56 ++++++++++---------- 4 files changed, 46 insertions(+), 42 deletions(-) diff --git a/app/Tdt/Core/Datasets/DatasetController.php b/app/Tdt/Core/Datasets/DatasetController.php index 181bc96c..bb6f05eb 100644 --- a/app/Tdt/Core/Datasets/DatasetController.php +++ b/app/Tdt/Core/Datasets/DatasetController.php @@ -281,22 +281,24 @@ private function processURI($uri) private static function applyRestFilter($data, $rest_params) { foreach ($rest_params as $rest_param) { - if (is_object($data) && $key = self::propertyExists($data, $rest_param)) { - $data = $data->$key; - } elseif (is_array($data)) { - if ($key = self::keyExists($data, $rest_param)) { - $data = $data[$key]; - } elseif (is_numeric($rest_param)) { - for ($i = 0; $i <= $rest_param; $i++) { - $result = array_shift($data); - } + if (!empty($rest_param)) { + if (is_object($data) && $key = self::propertyExists($data, $rest_param)) { + $data = $data->$key; + } elseif (is_array($data)) { + if ($key = self::keyExists($data, $rest_param)) { + $data = $data[$key]; + } elseif (is_numeric($rest_param)) { + for ($i = 0; $i <= $rest_param; $i++) { + $result = array_shift($data); + } - $data = $result; + $data = $result; + } else { + \App::abort(404, "No property ($rest_param) has been found."); + } } else { \App::abort(404, "No property ($rest_param) has been found."); } - } else { - \App::abort(404, "No property ($rest_param) has been found."); } } diff --git a/app/Tdt/Core/Formatters/GEOJSONFormatter.php b/app/Tdt/Core/Formatters/GEOJSONFormatter.php index cfd43216..b01c354d 100644 --- a/app/Tdt/Core/Formatters/GEOJSONFormatter.php +++ b/app/Tdt/Core/Formatters/GEOJSONFormatter.php @@ -74,6 +74,7 @@ public static function getBody($dataObj) 'geometry' => $geometric_ids[1], 'properties' => $dataRow ); + $id_prop = $dataObj->source_definition['map_property']; if (!empty($id_prop) && !empty($dataRow[$id_prop])) { $feature['id'] = $dataRow[$id_prop]; diff --git a/composer.json b/composer.json index 8d9427c3..7c66dc9c 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,7 @@ } }, "require": { - "laravel/framework": "4.2.0", + "laravel/framework": "4.2.1", "phpoffice/phpexcel": "1.8.0", "phpunit/phpunit" : "4.0.17", "easyrdf/easyrdf" : "0.9.1", @@ -30,7 +30,8 @@ "elasticsearch/elasticsearch": "1.4.1", "nesbot/carbon": "~1.14", "tdt/input" : "2.0.*", - "phayes/geophp" : "dev-master" + "phayes/geophp" : "dev-master", + "tdt/multisite" : "dev-master" }, "require-dev": { "mockery/mockery": "dev-master@dev" diff --git a/composer.lock b/composer.lock index b23ecc8d..bc5ef903 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "2d2ad64c882a005be73034f52be57824", + "hash": "edd5fd808bb935c036186a9a1b1b770b", "content-hash": "7cdec1bf315f21f0387492b52cb89aef", "packages": [ { @@ -802,22 +802,22 @@ }, { "name": "ml/json-ld", - "version": "1.0.3", + "version": "1.0.5", "target-dir": "ML/JsonLD", "source": { "type": "git", "url": "/~https://github.com/lanthaler/JsonLD.git", - "reference": "10a03f7dec0f4a7db6b2e6e6230cdfb7cce562a2" + "reference": "2f7f00a9daed844289135cc1cc99a75fc72a5438" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/lanthaler/JsonLD/zipball/10a03f7dec0f4a7db6b2e6e6230cdfb7cce562a2", - "reference": "10a03f7dec0f4a7db6b2e6e6230cdfb7cce562a2", + "url": "https://api.github.com/repos/lanthaler/JsonLD/zipball/2f7f00a9daed844289135cc1cc99a75fc72a5438", + "reference": "2f7f00a9daed844289135cc1cc99a75fc72a5438", "shasum": "" }, "require": { "ext-json": "*", - "ml/iri": ">=1.1.1", + "ml/iri": "^1.1.1", "php": ">=5.3.0" }, "require-dev": { @@ -847,7 +847,7 @@ "JSON-LD", "jsonld" ], - "time": "2014-09-15 10:41:19" + "time": "2016-01-17 17:39:22" }, { "name": "monolog/monolog", @@ -1865,7 +1865,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/proj4php/proj4php/zipball/6c9e3c7c7ad952a9f8381566efc2cb13b861588e", + "url": "https://api.github.com/repos/proj4php/proj4php/zipball/bcef74e906ffe5b9aab0e78b1e65143c0c2b5d42", "reference": "89722067010a4c88b7c4c79a7ee187bb2ddb1b61", "shasum": "" }, @@ -2583,16 +2583,16 @@ }, { "name": "symfony/event-dispatcher", - "version": "v2.8.0", + "version": "v2.8.2", "source": { "type": "git", "url": "/~https://github.com/symfony/event-dispatcher.git", - "reference": "a5eb815363c0388e83247e7e9853e5dbc14999cc" + "reference": "ee278f7c851533e58ca307f66305ccb9188aceda" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/a5eb815363c0388e83247e7e9853e5dbc14999cc", - "reference": "a5eb815363c0388e83247e7e9853e5dbc14999cc", + "url": "https://api.github.com/repos/symfony/event-dispatcher/zipball/ee278f7c851533e58ca307f66305ccb9188aceda", + "reference": "ee278f7c851533e58ca307f66305ccb9188aceda", "shasum": "" }, "require": { @@ -2639,20 +2639,20 @@ ], "description": "Symfony EventDispatcher Component", "homepage": "https://symfony.com", - "time": "2015-10-30 20:15:42" + "time": "2016-01-13 10:28:07" }, { "name": "symfony/filesystem", - "version": "v2.8.0", + "version": "v2.8.2", "source": { "type": "git", "url": "/~https://github.com/symfony/filesystem.git", - "reference": "3e661a0d521ac67496515fa6e6704bd61bcfff60" + "reference": "637b64d0ee10f44ae98dbad651b1ecdf35a11e8c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/filesystem/zipball/3e661a0d521ac67496515fa6e6704bd61bcfff60", - "reference": "3e661a0d521ac67496515fa6e6704bd61bcfff60", + "url": "https://api.github.com/repos/symfony/filesystem/zipball/637b64d0ee10f44ae98dbad651b1ecdf35a11e8c", + "reference": "637b64d0ee10f44ae98dbad651b1ecdf35a11e8c", "shasum": "" }, "require": { @@ -2688,7 +2688,7 @@ ], "description": "Symfony Filesystem Component", "homepage": "https://symfony.com", - "time": "2015-11-23 10:19:46" + "time": "2016-01-13 10:28:07" }, { "name": "symfony/finder", @@ -3100,16 +3100,16 @@ }, { "name": "symfony/yaml", - "version": "v2.8.0", + "version": "v2.8.2", "source": { "type": "git", "url": "/~https://github.com/symfony/yaml.git", - "reference": "f79824187de95064a2f5038904c4d7f0227fedb5" + "reference": "34c8a4b51e751e7ea869b8262f883d008a2b81b8" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/yaml/zipball/f79824187de95064a2f5038904c4d7f0227fedb5", - "reference": "f79824187de95064a2f5038904c4d7f0227fedb5", + "url": "https://api.github.com/repos/symfony/yaml/zipball/34c8a4b51e751e7ea869b8262f883d008a2b81b8", + "reference": "34c8a4b51e751e7ea869b8262f883d008a2b81b8", "shasum": "" }, "require": { @@ -3145,20 +3145,20 @@ ], "description": "Symfony Yaml Component", "homepage": "https://symfony.com", - "time": "2015-11-30 12:35:10" + "time": "2016-01-13 10:28:07" }, { "name": "tdt/input", - "version": "v2.0.0", + "version": "v2.0.1", "source": { "type": "git", "url": "/~https://github.com/tdt/input.git", - "reference": "1238eede3d9835abeaa6242ba33ef8bbe0e7bbd8" + "reference": "be4e71d42ff56c418121246460879cc4b0beae3a" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/tdt/input/zipball/1238eede3d9835abeaa6242ba33ef8bbe0e7bbd8", - "reference": "1238eede3d9835abeaa6242ba33ef8bbe0e7bbd8", + "url": "https://api.github.com/repos/tdt/input/zipball/be4e71d42ff56c418121246460879cc4b0beae3a", + "reference": "be4e71d42ff56c418121246460879cc4b0beae3a", "shasum": "" }, "require": { @@ -3198,7 +3198,7 @@ } }, "notification-url": "https://packagist.org/downloads/", - "time": "2015-12-09 15:31:06" + "time": "2016-01-09 14:58:53" }, { "name": "tdt/json", From ea43bdfc1462ba97954f0ca39bd5912c6e201774 Mon Sep 17 00:00:00 2001 From: Jan Vansteenlandt Date: Wed, 27 Jan 2016 10:59:34 +0100 Subject: [PATCH 2/5] Issue #373 and take into account corrupt line endings in a CSV file --- .../Core/DataControllers/CSVController.php | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/app/Tdt/Core/DataControllers/CSVController.php b/app/Tdt/Core/DataControllers/CSVController.php index 16fded6e..9a7622a1 100644 --- a/app/Tdt/Core/DataControllers/CSVController.php +++ b/app/Tdt/Core/DataControllers/CSVController.php @@ -6,6 +6,9 @@ use Tdt\Core\Pager; use Tdt\Core\Repositories\Interfaces\TabularColumnsRepositoryInterface; use Tdt\Core\Repositories\Interfaces\GeoPropertyRepositoryInterface; +use Tdt\Core\Datasets\DatasetController; + +ini_set('auto_detect_line_endings', true); /** * CSV Controller @@ -34,6 +37,15 @@ public function readData($source_definition, $rest_parameters = array()) { list($limit, $offset) = Pager::calculateLimitAndOffset(); + // Get the format, if it's CSV we allow a full read of the datasource without paging limitation + list($uri, $extension) = $this->processURI(\Request::path()); + + $ignore_paging = false; + + if (strtolower($extension) == 'csv') { + $ignore_paging = true; + } + // Disregard the paging when rest parameters are given if (!empty($rest_parameters)) { $limit = PHP_INT_MAX; @@ -130,7 +142,7 @@ public function readData($source_definition, $rest_parameters = array()) ); if (($handle = fopen($uri, "r", false, stream_context_create($ssl_options))) !== false) { - while (($data = fgetcsv($handle, 2000000, $delimiter)) !== false) { + while (($data = fgetcsv($handle, 0, $delimiter)) !== false) { if ($total_rows >= $start_row) { // Create the values array, containing the (aliased) name of the column // to the value of a the row which $data represents @@ -158,7 +170,7 @@ public function readData($source_definition, $rest_parameters = array()) $total_rows++; - if ($total_rows >= 10000) { + if ($total_rows >= 10000 && !$ignore_paging) { break; } } @@ -199,6 +211,41 @@ private function createValues($columns, $data) return $result; } + /** + * Parse the URI and look for the resource name and the format + * + * @param string $uri + * + * @return array + */ + private function processURI($uri) + { + $dot_position = strrpos($uri, '.'); + + if (!$dot_position) { + return array($uri, null); + } + + // If a dot has been found, do a couple + // of checks to find out if it introduces a formatter + $uri_parts = explode('.', $uri); + + $possible_extension = strtoupper(array_pop($uri_parts)); + + $uri = implode('.', $uri_parts); + + $formatter_class = 'Tdt\\Core\\Formatters\\' . $possible_extension . 'Formatter'; + + if (!class_exists($formatter_class)) { + // Re-attach the dot with the latter part of the uri + $uri .= '.' . strtolower($possible_extension); + + return array($uri, null); + } + + return array($uri, strtolower($possible_extension)); + } + /** * Parse the columns from a CSV file and return them * Optionally aliases can be given to columns as well as a primary key From 149bdf9bb28d9e451fed2f8ab77511a0036027a7 Mon Sep 17 00:00:00 2001 From: Erika Pauwels Date: Thu, 28 Jan 2016 15:09:49 +0100 Subject: [PATCH 3/5] Fix namespace of contactpoint in DCAT feed contactPoint is a property in the DCAT namespace according to the specifications. See https://www.w3.org/TR/vocab-dcat/#Property:dataset_contactPoint. --- app/Tdt/Core/Repositories/DcatRepository.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/Tdt/Core/Repositories/DcatRepository.php b/app/Tdt/Core/Repositories/DcatRepository.php index fbf4acaf..8f5ab302 100644 --- a/app/Tdt/Core/Repositories/DcatRepository.php +++ b/app/Tdt/Core/Repositories/DcatRepository.php @@ -37,8 +37,6 @@ public function getDcatDocument(array $definitions, $oldest_definition) // Create a new EasyRDF graph $graph = new \EasyRdf_Graph(); - \EasyRdf_Namespace::set('adms', 'http://www.w3.org/ns/adms#'); - $all_settings = $this->settings->getAll(); $uri = \Request::root(); @@ -105,7 +103,7 @@ public function getDcatDocument(array $definitions, $oldest_definition) // Backwards compatibility if (!empty($definition['contact_point'])) { - $graph->addResource($dataset_uri, 'adms:contactPoint', $definition['contact_point']); + $graph->addResource($dataset_uri, 'dcat:contactPoint', $definition['contact_point']); } // Add the publisher resource to the dataset From 34b25a1d14df27e9a456e2d56a8cdf5f213ed3fb Mon Sep 17 00:00:00 2001 From: Jan Vansteenlandt Date: Mon, 1 Feb 2016 10:25:34 +0100 Subject: [PATCH 4/5] Only perform a HEAD check instead of fetching an entire file to check its existence --- .../Core/DataControllers/CSVController.php | 1 - app/Tdt/Core/Validators/CustomValidator.php | 34 ++++++++++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/app/Tdt/Core/DataControllers/CSVController.php b/app/Tdt/Core/DataControllers/CSVController.php index 9a7622a1..e1a076b4 100644 --- a/app/Tdt/Core/DataControllers/CSVController.php +++ b/app/Tdt/Core/DataControllers/CSVController.php @@ -175,7 +175,6 @@ public function readData($source_definition, $rest_parameters = array()) } } fclose($handle); - } else { \App::abort(500, "Cannot retrieve any data from the CSV file on location $uri."); } diff --git a/app/Tdt/Core/Validators/CustomValidator.php b/app/Tdt/Core/Validators/CustomValidator.php index cd1d35db..70f2c41c 100644 --- a/app/Tdt/Core/Validators/CustomValidator.php +++ b/app/Tdt/Core/Validators/CustomValidator.php @@ -10,17 +10,18 @@ */ class CustomValidator extends \Illuminate\Validation\Validator { - /** * Check if the URI can be resolved externally or locally */ public function validateUri($attribute, $value, $parameters) { try { - if (!filter_var($value, FILTER_VALIDATE_URL) === false) { - $data = $this->getRemoteData($value); + $url_pieces = parse_url($value); - return !empty($data); + if (!filter_var($value, FILTER_VALIDATE_URL) === false && ($url_pieces['scheme'] == 'http' || $url_pieces['scheme'] == 'https')) { + $status = $this->getHeadInfo($value); + + return $status == 200; } else { $data =@ file_get_contents($value); @@ -31,6 +32,31 @@ public function validateUri($attribute, $value, $parameters) } } + private function getHeadInfo($uri) + { + $c = curl_init(); + curl_setopt($c, CURLOPT_URL, $uri); + curl_setopt($c, CURLOPT_RETURNTRANSFER, 1); + + curl_setopt($c, CURLOPT_SSL_VERIFYHOST, false); + curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false); + + curl_setopt($c, CURLOPT_CONNECTTIMEOUT, 9); + curl_setopt($c, CURLOPT_TIMEOUT, 60); + curl_setopt($c, CURLOPT_CUSTOMREQUEST, 'HEAD'); + curl_setopt($c, CURLOPT_NOBODY, true); + + curl_exec($c); + $status = curl_getinfo($c); + curl_close($c); + + if (!empty($status['http_code'])) { + return $status['http_code']; + } else { + return 500; + } + } + private function getRemoteData($url) { $c = curl_init(); From 82b1583e26e0bbc56b1b09be09ba4500f47d1f5a Mon Sep 17 00:00:00 2001 From: Jan Vansteenlandt Date: Mon, 1 Feb 2016 10:27:41 +0100 Subject: [PATCH 5/5] Remove testing dependency --- composer.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 7c66dc9c..14ed59b2 100644 --- a/composer.json +++ b/composer.json @@ -30,8 +30,7 @@ "elasticsearch/elasticsearch": "1.4.1", "nesbot/carbon": "~1.14", "tdt/input" : "2.0.*", - "phayes/geophp" : "dev-master", - "tdt/multisite" : "dev-master" + "phayes/geophp" : "dev-master" }, "require-dev": { "mockery/mockery": "dev-master@dev"