From 45973125060ce3e16740759e617ae768348599b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20B=C3=B6sch-Plepelits?= Date: Mon, 18 Apr 2022 11:07:42 +0200 Subject: [PATCH 01/10] AJAX fun 'wikipedia': accept wikidata id as value --- src/wikipedia.php | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/src/wikipedia.php b/src/wikipedia.php index 54052567..a36c9920 100644 --- a/src/wikipedia.php +++ b/src/wikipedia.php @@ -4,12 +4,48 @@ function ajax_wikipedia ($param) { $wp_lang = $m[1]; $wp_page = $m[2]; } + elseif (preg_match("/^Q\d+$/", $param['page'])) { + $id = $param['page']; + $body = file_get_contents("https://www.wikidata.org/wiki/Special:EntityData/{$id}.json"); + $body = json_decode($body, true); - if (!isset($wp_lang) || !isset($wp_page)) { + if (!array_key_exists('entities', $body) || !array_key_exists($id, $body['entities'])) { + return false; + } + + $data = $body['entities'][$id]; + + if (array_key_exists('sitelinks', $data)) { + if (array_key_exists($param['lang'] . 'wiki', $data['sitelinks'])) { + $wp_lang = $param['lang']; + $wp_url = $data['sitelinks'][$param['lang'] . 'wiki']['url']; + } + elseif (array_key_exists('enwiki', $data['sitelinks'])) { + $wp_lang = 'en'; + $wp_url = $data['sitelinks']['enwiki']['url']; + } + else { + $sitelinks_ids = array_keys($data['sitelinks']); + $sitelinks_ids = array_values(array_filter($sitelinks_ids, function ($id) { + return preg_match('/wiki$/', $id); + })); + + if (sizeof($sitelinks_ids)) { + $id = $sitelinks_ids[0]; + $wp_lang = substr($id, 0, strlen($id) - 4); + $wp_url = $data['sitelinks'][$id]['url']; + } + } + } + } + + if (!isset($wp_lang) || !(isset($wp_page) || isset($wp_url))) { return false; } - $wp_url = "https://{$wp_lang}.wikipedia.org/wiki/" . urlencode(strtr($wp_page, array(" " => "_"))); + if (!isset($wp_url)) { + $wp_url = "https://{$wp_lang}.wikipedia.org/wiki/" . urlencode(strtr($wp_page, array(" " => "_"))); + } $content = file_get_contents($wp_url); From 510dd2d41f76e40b5d14c633b0a155d4f9d6e1a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20B=C3=B6sch-Plepelits?= Date: Tue, 19 Apr 2022 09:55:00 +0200 Subject: [PATCH 02/10] wikipedia: cache result of getAbstract(); return result synchronizly if known --- src/wikipedia.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/wikipedia.js b/src/wikipedia.js index 26f918cc..9cabb9e7 100644 --- a/src/wikipedia.js +++ b/src/wikipedia.js @@ -2,6 +2,7 @@ var wikidata = require('./wikidata') const displayBlock = require('./displayBlock') var cache = {} +var getAbstractCache = {} var loadClash = {} function stripLinks (dom) { @@ -98,6 +99,11 @@ function get (value, callback) { } function getAbstract (value, callback) { + if (value in getAbstractCache) { + callback(null, getAbstractCache[value]) + return getAbstractCache[value] + } + get(value, function (err, result) { var text = null From 80e547294e889506b96eadddd6810297c07f799a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20B=C3=B6sch-Plepelits?= Date: Tue, 19 Apr 2022 10:07:14 +0200 Subject: [PATCH 03/10] wikipedia: new twig filter 'wikipediaAbstract' --- doc/TwigJS.md | 1 + src/wikipedia.js | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/doc/TwigJS.md b/doc/TwigJS.md index 5ed56f2a..6e21ed3d 100644 --- a/doc/TwigJS.md +++ b/doc/TwigJS.md @@ -76,6 +76,7 @@ Extra filters: * filter `md5`: calculate md5 hash of a string. * filter `enumerate`: enumerate the given list, e.g. "foo, bar, and bla". Input either an array (`[ "foo", "bar", "bla" ]|enumerate`) or a string with `;` as separator (`"foo;bar;bla"|enumerate`). * filter `debug`: print the value (and further arguments) to the javascript console (via `console.log()`) +* filter `wikipediaAbstract`: shows the abstract of a Wikipedia article in the selected data language (or, if not available, the language which was used in input, resp. 'en' for Wikidata input). Input is either 'language:article' (e.g. 'en:Douglas Adams') or a wikidata id (e.g. 'Q42'). Notes: * Variables will automatically be HTML escaped, if not the filter raw is used, e.g.: {{ tags.name|raw }} diff --git a/src/wikipedia.js b/src/wikipedia.js index 9cabb9e7..d99cf851 100644 --- a/src/wikipedia.js +++ b/src/wikipedia.js @@ -1,3 +1,6 @@ +const async = require('async') +const OverpassLayer = require('overpass-layer') + var wikidata = require('./wikidata') const displayBlock = require('./displayBlock') @@ -116,11 +119,43 @@ function getAbstract (value, callback) { text += ' ' + lang('more') + '' } + getAbstractCache[value] = text + callback(err, text) } ) } +function updateDomWikipedia (dom, callback) { + const wikipediaQueries = dom.querySelectorAll('.wikipedia') + async.each( + wikipediaQueries, + (div, done) => { + if (div.hasAttribute('data-done')) { + return done() + } + + getAbstract(div.getAttribute('data-id'), + (err, result) => { + if (result) { + div.innerHTML = result + div.setAttribute('data-done', 'true') + } + done() + } + ) + }, + () => { + callback() + } + ) +} + +register_hook('show-popup', function (data, category, dom, callback) { + updateDomWikipedia(dom, () => updateDomWikipedia(dom, () => {})) + callback() +}) + register_hook('show-details', function (data, category, dom, callback) { var ob = data.object var found = 0 @@ -381,3 +416,12 @@ function getImages (tagValue, callback) { module.exports = { getImages: getImages } + +OverpassLayer.twig.extendFilter('wikipediaAbstract', function (value, param) { + if (value in getAbstractCache) { + let result = getAbstractCache[value] + return '
' + result + '
' + } + + return '
' + value + '
' +}) From 161ee85ec96d9fa28740be6f416104d7f11550b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20B=C3=B6sch-Plepelits?= Date: Tue, 19 Apr 2022 10:26:54 +0200 Subject: [PATCH 04/10] wikidata/wikipedia: function to load a wikidata item --- modulekit.php | 1 + src/wikidata.php | 19 +++++++++++++++++++ src/wikipedia.php | 10 +--------- 3 files changed, 21 insertions(+), 9 deletions(-) create mode 100644 src/wikidata.php diff --git a/modulekit.php b/modulekit.php index 669a9fd8..0dfd431d 100644 --- a/modulekit.php +++ b/modulekit.php @@ -15,6 +15,7 @@ $include = array( 'src/options.php', 'src/language.php', 'src/ip-location.php', + 'src/wikidata.php', 'src/wikipedia.php', 'src/ImageLoader.php', 'src/RepositoryBase.php', diff --git a/src/wikidata.php b/src/wikidata.php new file mode 100644 index 00000000..28d941f5 --- /dev/null +++ b/src/wikidata.php @@ -0,0 +1,19 @@ + Date: Tue, 19 Apr 2022 21:25:26 +0200 Subject: [PATCH 05/10] wikidata.getAbstract(): include language in cacheId --- src/wikipedia.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/wikipedia.js b/src/wikipedia.js index d99cf851..ff80c381 100644 --- a/src/wikipedia.js +++ b/src/wikipedia.js @@ -102,9 +102,10 @@ function get (value, callback) { } function getAbstract (value, callback) { - if (value in getAbstractCache) { - callback(null, getAbstractCache[value]) - return getAbstractCache[value] + const cacheId = options.data_lang + ':' + value + if (cacheId in getAbstractCache) { + callback(null, getAbstractCache[cacheId]) + return getAbstractCache[cacheId] } get(value, @@ -119,7 +120,7 @@ function getAbstract (value, callback) { text += ' ' + lang('more') + '' } - getAbstractCache[value] = text + getAbstractCache[cacheId] = text callback(err, text) } @@ -418,8 +419,9 @@ module.exports = { } OverpassLayer.twig.extendFilter('wikipediaAbstract', function (value, param) { - if (value in getAbstractCache) { - let result = getAbstractCache[value] + const cacheId = options.data_lang + ':' + value + if (cacheId in getAbstractCache) { + let result = getAbstractCache[cacheId] return '
' + result + '
' } From c5154dbcfe08025e89dfd8335ee9f90a73f2b1a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20B=C3=B6sch-Plepelits?= Date: Tue, 19 Apr 2022 21:29:41 +0200 Subject: [PATCH 06/10] wikidata/wikipedia: if there's no wikipedia abstract, show the label from Wikidata; url to commons or wikidata --- src/wikidata.php | 17 +++++++++++++++++ src/wikipedia.php | 31 ++++++++++++++++++++++--------- 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/wikidata.php b/src/wikidata.php index 28d941f5..52c74d19 100644 --- a/src/wikidata.php +++ b/src/wikidata.php @@ -17,3 +17,20 @@ function wikidataLoad ($id) { return $wikidataCache[$id]; } + +function wikidataGetLabel ($id, $lang) { + $data = wikidataLoad($id); + + if (array_key_exists($lang, $data['labels'])) { + return $data['labels'][$lang]['value']; + } + elseif (array_key_exists('en', $data['labels'])) { + return $data['labels']['en']['value']; + } + elseif (!sizeof($data['labels'])) { + return $id; + } + else { + return array_values($data['labels'])[0]['value']; + } +} diff --git a/src/wikipedia.php b/src/wikipedia.php index 501ed9a8..3f42dcdc 100644 --- a/src/wikipedia.php +++ b/src/wikipedia.php @@ -8,6 +8,11 @@ function ajax_wikipedia ($param) { $data = wikidataLoad($param['page']); if (array_key_exists('sitelinks', $data)) { + $sitelinks_ids = array_keys($data['sitelinks']); + $sitelinks_ids = array_values(array_filter($sitelinks_ids, function ($id) { + return preg_match('/wiki$/', $id) && $id !== 'commonswiki'; + })); + if (array_key_exists($param['lang'] . 'wiki', $data['sitelinks'])) { $wp_lang = $param['lang']; $wp_url = $data['sitelinks'][$param['lang'] . 'wiki']['url']; @@ -16,17 +21,25 @@ function ajax_wikipedia ($param) { $wp_lang = 'en'; $wp_url = $data['sitelinks']['enwiki']['url']; } + elseif (sizeof($sitelinks_ids)) { + $id = $sitelinks_ids[0]; + $wp_lang = substr($id, 0, strlen($id) - 4); + $wp_url = $data['sitelinks'][$id]['url']; + } else { - $sitelinks_ids = array_keys($data['sitelinks']); - $sitelinks_ids = array_values(array_filter($sitelinks_ids, function ($id) { - return preg_match('/wiki$/', $id); - })); - - if (sizeof($sitelinks_ids)) { - $id = $sitelinks_ids[0]; - $wp_lang = substr($id, 0, strlen($id) - 4); - $wp_url = $data['sitelinks'][$id]['url']; + $content = "

" . wikidataGetLabel($param['page'], $param['lang']) . "

"; + $url = "https://wikidata.org/wiki/{$param['page']}"; + if (array_key_exists('commonswiki', $data['sitelinks'])) { + $url = $data['sitelinks']['commonswiki']['url']; } + + return array( + 'content' => $content, + 'languages' => [ + $param['lang'] => $url, + ], + 'language' => $param['lang'], + ); } } } From 531192a7be24d479a5f0d41dee1d8837507e37a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20B=C3=B6sch-Plepelits?= Date: Tue, 19 Apr 2022 21:30:34 +0200 Subject: [PATCH 07/10] wikidata/wikipedia: create formatted entry from wikidata --- src/wikidata.php | 62 +++++++++++++++++++++++++++++++++++++++++++++++ src/wikipedia.php | 2 +- 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/wikidata.php b/src/wikidata.php index 52c74d19..4d1c3a25 100644 --- a/src/wikidata.php +++ b/src/wikidata.php @@ -34,3 +34,65 @@ function wikidataGetLabel ($id, $lang) { return array_values($data['labels'])[0]['value']; } } + +function wikidataGetValues ($id, $property) { + $data = wikidataLoad($id); + + if (!array_key_exists($property, $data['claims'])) { + return []; + } + + return array_map( + function ($el) { + return $el['mainsnak']['datavalue']['value']; + }, + $data['claims'][$property] + ); +} + +function wikidataFormatDate($value, $maxPrecision = 13) { + $v = new DateTime($value['time']); + $p = min($maxPrecision, $value['precision']); + + if ($p < 9) { + } else { + $formats = [ + 9 => 'Y', + 10 => 'M Y', + 11 => 'j. M Y', + 12 => 'j. M Y - G:00', + 13 => 'j. M Y - G:i', + 14 => 'j. M Y - G:i:s', + ]; + + return $v->format($formats[$p]); + } +} + +function wikidataFormat ($id, $lang) { + $ret = '' . wikidataGetLabel($id, $lang) . ''; + + $birthDate = wikidataGetValues($id, 'P569'); + $deathDate = wikidataGetValues($id, 'P570'); + if (sizeof($birthDate) && sizeof($deathDate)) { + $ret .= ' (' . wikidataFormatDate($birthDate[0], 11) . ' — ' . wikidataFormatDate($deathDate[0], 11) . ')'; + } + elseif (sizeof($birthDate)) { + $ret .= ' (* ' . wikidataFormatDate($birthDate[0], 11) . ')'; + } + elseif (sizeof($deathDate)) { + $ret .= ' († ' . wikidataFormatDate($birthDate[0], 11) . ')'; + } + + $occupation = wikidataGetValues($id, 'P106'); + if (sizeof($occupation)) { + $ret .= ', ' . implode(', ', array_map( + function ($value) use ($lang) { + return wikidataGetLabel($value['id'], $lang); + }, + $occupation + )); + } + + return $ret; +} diff --git a/src/wikipedia.php b/src/wikipedia.php index 3f42dcdc..e34eed56 100644 --- a/src/wikipedia.php +++ b/src/wikipedia.php @@ -27,7 +27,7 @@ function ajax_wikipedia ($param) { $wp_url = $data['sitelinks'][$id]['url']; } else { - $content = "

" . wikidataGetLabel($param['page'], $param['lang']) . "

"; + $content = "

" . wikidataFormat($param['page'], $param['lang']) . "

"; $url = "https://wikidata.org/wiki/{$param['page']}"; if (array_key_exists('commonswiki', $data['sitelinks'])) { $url = $data['sitelinks']['commonswiki']['url']; From 04883f2ee238dfe4e75dd49ef177c36cb79a931c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20B=C3=B6sch-Plepelits?= Date: Mon, 6 Jun 2022 13:20:46 +0200 Subject: [PATCH 08/10] TwigJS: improve wording --- doc/TwigJS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/TwigJS.md b/doc/TwigJS.md index 6e21ed3d..ec9e2f9e 100644 --- a/doc/TwigJS.md +++ b/doc/TwigJS.md @@ -79,6 +79,6 @@ Extra filters: * filter `wikipediaAbstract`: shows the abstract of a Wikipedia article in the selected data language (or, if not available, the language which was used in input, resp. 'en' for Wikidata input). Input is either 'language:article' (e.g. 'en:Douglas Adams') or a wikidata id (e.g. 'Q42'). Notes: -* Variables will automatically be HTML escaped, if not the filter raw is used, e.g.: {{ tags.name|raw }} +* Variables will automatically be HTML escaped, unless the filter raw is used, e.g.: {{ tags.name|raw }} * The templates will be rendered when the object becomes visible and when the zoom level changes. * If you set an arbitrary value within a twig template (e.g.: {% set foo = "bar" %}), it will also be available in further templates of the same object by using (e.g.: {{ foo }}). The templates will be evaluated in the order as they are defined. From 1a270fb71fdda73b9ce71d56911f57e9151dd51b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20B=C3=B6sch-Plepelits?= Date: Tue, 21 Jun 2022 20:33:53 +0200 Subject: [PATCH 09/10] Wikidata/TwigJS: filter 'wikidataEntity' returns structured data of the object --- doc/TwigJS.md | 1 + src/wikidata.js | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/doc/TwigJS.md b/doc/TwigJS.md index ec9e2f9e..6eeb0a23 100644 --- a/doc/TwigJS.md +++ b/doc/TwigJS.md @@ -77,6 +77,7 @@ Extra filters: * filter `enumerate`: enumerate the given list, e.g. "foo, bar, and bla". Input either an array (`[ "foo", "bar", "bla" ]|enumerate`) or a string with `;` as separator (`"foo;bar;bla"|enumerate`). * filter `debug`: print the value (and further arguments) to the javascript console (via `console.log()`) * filter `wikipediaAbstract`: shows the abstract of a Wikipedia article in the selected data language (or, if not available, the language which was used in input, resp. 'en' for Wikidata input). Input is either 'language:article' (e.g. 'en:Douglas Adams') or a wikidata id (e.g. 'Q42'). +* filter `wikidataEntity`: returns the wikidata entity in structured form (or `null` if the entity is not cached or `false` if it does not exist). Example: https://www.wikidata.org/wiki/Special:EntityData/Q42.json Notes: * Variables will automatically be HTML escaped, unless the filter raw is used, e.g.: {{ tags.name|raw }} diff --git a/src/wikidata.js b/src/wikidata.js index fe7da3ab..31abfc68 100644 --- a/src/wikidata.js +++ b/src/wikidata.js @@ -1,3 +1,5 @@ +const OverpassLayer = require('overpass-layer') + var httpGet = require('./httpGet') var loadClash = {} var cache = {} @@ -22,6 +24,7 @@ function wikidataLoad (id, callback) { if (!result.entities || !result.entities[id]) { console.log('invalid result', result) + cache[id] = false return callback(err, null) } @@ -39,3 +42,13 @@ function wikidataLoad (id, callback) { module.exports = { load: wikidataLoad } + +OverpassLayer.twig.extendFilter('wikidataEntity', function (value, param) { + if (value in cache) { + return cache[value] + } + + wikidataLoad(value, () => {}) + + return null +}) From 9245945c532e6d2ac234c485a6a5162a6a5fb033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20B=C3=B6sch-Plepelits?= Date: Wed, 22 Jun 2022 19:34:59 +0200 Subject: [PATCH 10/10] wikipedia: filter wikipediaAbstract returns escape html code --- src/wikipedia.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/wikipedia.js b/src/wikipedia.js index ff80c381..aa2a94ec 100644 --- a/src/wikipedia.js +++ b/src/wikipedia.js @@ -419,11 +419,14 @@ module.exports = { } OverpassLayer.twig.extendFilter('wikipediaAbstract', function (value, param) { + let result const cacheId = options.data_lang + ':' + value if (cacheId in getAbstractCache) { - let result = getAbstractCache[cacheId] - return '
' + result + '
' + const text = getAbstractCache[cacheId] + result = '
' + text + '
' + } else { + result = '
' + value + '
' } - return '
' + value + '
' + return OverpassLayer.twig.filters.raw(result) })