Commit 03e23a28 authored by Benjamin Bellamy's avatar Benjamin Bellamy 💬 Committed by Yassine Doghri
Browse files

feat: update analytics so to meet IABv2 requirements

- https://iabtechlab.com/wp-content/uploads/2017/12/Podcast_Measurement_v2-Dec-20-2017.pdf
  - no IP address is ever stored on the server. Only aggregate data is stored in the dababase.
  - rolling 24-hour window
  - castopod does not do pre-load
  - IP Blacklisting https://github.com/client9/ipcat
  - user-agent Filtering https://github.com/opawg/user-agents
  - ignores 2 bytes range "Range: 0-1" (performed by official Apple iOS Podcast app)
  - in case of partial content, adds up all requests to check >1mn was downloaded
  - identifying Uniques is done with a combination of IP Address and User Agent
- add AMcharts
- add some graphs
- add regions to analytics
- add ipcat blacklist
- enhance useragents performances
- add filesize and header size in order to calculate 1mn downloads
- update publisher ID3 field
- update castopod icon
- add disclaimer and warning import form translation
- update docs/setup-development.md

closes #10
parent 5417be00
......@@ -39,13 +39,11 @@ bundle_app:
script:
# build all assets for views
- npm run build
# download GeoLite2-Country and opawg/user-agents archives and extract them to writable/uploads
- wget -c "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=$MAXMIND_LICENCE_KEY&suffix=tar.gz" -O - | tar -xz -C ./writable/uploads/
- wget -c "https://github.com/opawg/user-agents/archive/master.tar.gz" -O - | tar -xz -C ./writable/uploads/
# download GeoLite2-City archive and extract it to writable/uploads
- wget -c "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=$MAXMIND_LICENCE_KEY&suffix=tar.gz" -O - | tar -xz -C ./writable/uploads/
# rename extracted archives' folders
- mv ./writable/uploads/GeoLite2-Country* ./writable/uploads/GeoLite2-Country
- mv ./writable/uploads/user-agents* ./writable/uploads/user-agents
- mv ./writable/uploads/GeoLite2-City* ./writable/uploads/GeoLite2-City
# create bundle folder: uses .rsync-filter (-F) file to copy only needed files
- rsync -avF --progress . ./bundle
......
......@@ -12,16 +12,20 @@ PHP Dependencies:
- [commonmark](https://commonmark.thephpleague.com/) ([BSD 3-Clause "New" or "Revised" License](https://github.com/thephpleague/commonmark/blob/latest/LICENSE))
- [phpdotenv](https://github.com/vlucas/phpdotenv) ([ BSD-3-Clause License ](https://github.com/vlucas/phpdotenv/blob/master/LICENSE))
- [HTML To Markdown for PHP](https://github.com/thephpleague/html-to-markdown) ([MIT License](https://github.com/thephpleague/html-to-markdown/blob/master/LICENSE))
- [podlibre/user-agents-php](https://github.com/podlibre/user-agents-php) ([MIT License](https://github.com/podlibre/user-agents-php/blob/main/LICENSE))
- [podlibre/ipcat](https://github.com/podlibre/ipcat) ([GNU General Public License v3.0](https://github.com/podlibre/ipcat/blob/master/LICENSE))
Javascript dependencies:
- [rollup](https://rollupjs.org/) ([MIT License](https://github.com/rollup/rollup/blob/master/LICENSE.md))
- [tailwindcss](https://tailwindcss.com/) ([MIT License](https://github.com/tailwindcss/tailwindcss/blob/master/LICENSE))
- [ProseMirror](https://prosemirror.net/) ([MIT License](https://github.com/ProseMirror/prosemirror/blob/master/LICENSE))
- [D3: Data-Driven Documents](https://d3js.org) ([BSD 3-Clause "New" or "Revised" License](https://github.com/d3/d3/blob/master/LICENSE))
- [amCharts 4](https://github.com/amcharts/amcharts4) ([Free amCharts license](https://github.com/amcharts/amcharts4/blob/master/dist/script/LICENSE))
- [Choices.js](https://joshuajohnson.co.uk/Choices/) ([MIT License](https://github.com/jshjohnson/Choices/blob/master/LICENSE))
Other:
- [RemixIcon](https://remixicon.com/) ([Apache License 2.0](https://github.com/Remix-Design/RemixIcon/blob/master/License))
- [User agent list](https://github.com/opawg/user-agents) ([by Open Podcast Analytics Working Group](https://github.com/opawg)) ([MIT license](https://github.com/opawg/user-agents/blob/master/LICENSE))
- [OPAWG/User agent list](https://github.com/opawg/user-agents) ([by Open Podcast Analytics Working Group](https://github.com/opawg)) ([MIT license](https://github.com/opawg/user-agents/blob/master/LICENSE))
- [client9/ipcat](https://github.com/client9/ipcat) ([GNU General Public License v3.0](https://github.com/client9/ipcat/blob/master/LICENSE))
- [GeoLite2 City](https://dev.maxmind.com/geoip/geoip2/geolite2/) ([Attribution-ShareAlike 4.0 International (CC BY-SA 4.0)](https://www.maxmind.com/en/geolite2/eula))
......@@ -53,10 +53,14 @@ $routes->group(config('App')->installGateway, function ($routes) {
]);
});
// Route for podcast audio file analytics (/stats/podcast_id/episode_id/podcast_folder/filename.mp3)
$routes->add('stats/(:num)/(:num)/(:any)', 'Analytics::hit/$1/$2/$3', [
'as' => 'analytics_hit',
]);
// Route for podcast audio file analytics (/audio/podcast_id/episode_id/bytes_threshold/filesize/podcast_folder/filename.mp3)
$routes->add(
'audio/(:num)/(:num)/(:num)/(:num)/(:any)',
'Analytics::hit/$1/$2/$3/$4/$5',
[
'as' => 'analytics_hit',
]
);
// Show the Unknown UserAgents
$routes->get('.well-known/unknown-useragents', 'UnknownUserAgents');
......@@ -113,6 +117,26 @@ $routes->group(
'as' => 'podcast-delete',
'filter' => 'permission:podcasts-delete',
]);
$routes->get('analytics', 'Podcast::analytics/$1', [
'as' => 'podcast-analytics',
'filter' => 'permission:podcasts-view,podcast-view',
]);
$routes->get(
'analytics-data/(:segment)/(:segment)',
'AnalyticsData::getData/$1/$2/$3',
[
'as' => 'analytics-data',
'filter' => 'permission:podcasts-view,podcast-view',
]
);
$routes->get(
'analytics-data/(:segment)/(:segment)/(:num)',
'AnalyticsData::getData/$1/$2/$3/$4',
[
'as' => 'analytics-filtered-data',
'filter' => 'permission:podcasts-view,podcast-view',
]
);
// Podcast episodes
$routes->group('episodes', function ($routes) {
......
<?php
/**
* @copyright 2020 Podlibre
* @license https://www.gnu.org/licenses/agpl-3.0.en.html AGPL3
* @link https://castopod.org/
*/
namespace App\Controllers\Admin;
use App\Models\PodcastModel;
use App\Models\EpisodeModel;
class AnalyticsData extends BaseController
{
/**
* @var \App\Entities\Podcast|null
*/
protected $podcast;
protected $className;
protected $methodName;
protected $episode;
public function _remap($method, ...$params)
{
if (count($params) > 2) {
if (!($this->podcast = (new PodcastModel())->find($params[0]))) {
throw \CodeIgniter\Exceptions\PageNotFoundException::forPageNotFound(
'Podcast not found: ' . $params[0]
);
}
$this->className = '\App\Models\Analytics' . $params[1] . 'Model';
$this->methodName = 'getData' . $params[2];
if (count($params) > 3) {
if (
!($this->episode = (new EpisodeModel())
->where([
'podcast_id' => $this->podcast->id,
'id' => $params[3],
])
->first())
) {
throw \CodeIgniter\Exceptions\PageNotFoundException::forPageNotFound(
'Episode not found: ' . $params[3]
);
}
}
}
return $this->$method();
}
public function getData()
{
$analytics_model = new $this->className();
$methodName = $this->methodName;
if ($this->episode) {
return $this->response->setJSON(
$analytics_model->$methodName(
$this->podcast->id,
$this->episode->id
)
);
} else {
return $this->response->setJSON(
$analytics_model->$methodName($this->podcast->id)
);
}
}
}
......@@ -58,6 +58,14 @@ class Podcast extends BaseController
return view('admin/podcast/view', $data);
}
public function analytics()
{
$data = ['podcast' => $this->podcast];
replace_breadcrumb_params([0 => $this->podcast->title]);
return view('admin/podcast/analytics', $data);
}
public function create()
{
helper(['form', 'misc']);
......@@ -204,7 +212,9 @@ class Podcast extends BaseController
$podcast = new \App\Entities\Podcast([
'name' => $this->request->getPost('name'),
'imported_feed_url' => $this->request->getPost('imported_feed_url'),
'new_feed_url' => base_url(
route_to('podcast_feed', $this->request->getPost('name'))
),
'title' => $feed->channel[0]->title,
'description' => $feed->channel[0]->description,
'image' => download_file($nsItunes->image->attributes()),
......@@ -214,7 +224,9 @@ class Podcast extends BaseController
? null
: (in_array($nsItunes->explicit, ['yes', 'true'])
? 'explicit'
: null),
: (in_array($nsItunes->explicit, ['no', 'false'])
? 'clean'
: null)),
'owner_name' => $nsItunes->owner->name,
'owner_email' => $nsItunes->owner->email,
'publisher' => $nsItunes->author,
......@@ -302,11 +314,13 @@ class Podcast extends BaseController
'image' => empty($nsItunes->image->attributes())
? null
: download_file($nsItunes->image->attributes()),
'explicit' => $nsItunes->explicit
? (in_array($nsItunes->explicit, ['yes', 'true'])
'parental_advisory' => empty($nsItunes->explicit)
? null
: (in_array($nsItunes->explicit, ['yes', 'true'])
? 'explicit'
: null)
: null,
: (in_array($nsItunes->explicit, ['no', 'false'])
? 'clean'
: null)),
'number' =>
$this->request->getPost('force_renumber') === 'yes'
? $itemNumber
......
......@@ -40,16 +40,22 @@ class Analytics extends Controller
// E.g.:
// $this->session = \Config\Services::session();
set_user_session_country();
set_user_session_deny_list_ip();
set_user_session_location();
set_user_session_player();
}
// Add one hit to this episode:
public function hit($p_podcastId, $p_episodeId, ...$filename)
{
public function hit(
$podcastId,
$episodeId,
$bytesThreshold,
$fileSize,
...$filename
) {
helper('media');
podcast_hit($p_podcastId, $p_episodeId);
podcast_hit($podcastId, $episodeId, $bytesThreshold, $fileSize);
return redirect()->to(media_url(implode('/', $filename)));
}
}
......@@ -45,9 +45,10 @@ class BaseController extends Controller
// E.g.:
// $this->session = \Config\Services::session();
set_user_session_country();
set_user_session_deny_list_ip();
set_user_session_browser();
set_user_session_referer();
set_user_session_entry_page();
}
protected static function triggerWebpageHit($podcastId)
......
......@@ -110,6 +110,13 @@ class AddPodcasts extends Migration
'The RSS feed URL if this podcast was imported, NULL otherwise.',
'null' => true,
],
'new_feed_url' => [
'type' => 'VARCHAR',
'constraint' => 1024,
'comment' =>
'The RSS new feed URL if this podcast is moving out, NULL otherwise.',
'null' => true,
],
'created_at' => [
'type' => 'TIMESTAMP',
],
......
......@@ -61,6 +61,12 @@ class AddEpisodes extends Migration
'unsigned' => true,
'comment' => 'File size in bytes',
],
'enclosure_headersize' => [
'type' => 'INT',
'constraint' => 10,
'unsigned' => true,
'comment' => 'Header size in bytes',
],
'description' => [
'type' => 'TEXT',
'null' => true,
......
<?php
/**
* Class AddAnalyticsPodcastsByCountry
* Creates analytics_podcasts_by_country table in database
* @copyright 2020 Podlibre
* @license https://www.gnu.org/licenses/agpl-3.0.en.html AGPL3
* @link https://castopod.org/
*/
namespace App\Database\Migrations;
use CodeIgniter\Database\Migration;
class AddAnalyticsPodcasts extends Migration
{
public function up()
{
$this->forge->addField([
'podcast_id' => [
'type' => 'BIGINT',
'constraint' => 20,
'unsigned' => true,
],
'date' => [
'type' => 'date',
],
'hits' => [
'type' => 'INT',
'constraint' => 10,
'default' => 1,
],
]);
$this->forge->addPrimaryKey(['podcast_id', 'date']);
$this->forge->addField(
'`created_at` timestamp NOT NULL DEFAULT current_timestamp()'
);
$this->forge->addField(
'`updated_at` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp()'
);
$this->forge->addForeignKey('podcast_id', 'podcasts', 'id');
$this->forge->createTable('analytics_podcasts');
}
public function down()
{
$this->forge->dropTable('analytics_podcasts');
}
}
......@@ -12,34 +12,28 @@ namespace App\Database\Migrations;
use CodeIgniter\Database\Migration;
class AddAnalyticsEpisodesByCountry extends Migration
class AddAnalyticsPodcastsByEpisode extends Migration
{
public function up()
{
$this->forge->addField([
'id' => [
'type' => 'BIGINT',
'constraint' => 20,
'unsigned' => true,
'auto_increment' => true,
],
'podcast_id' => [
'type' => 'BIGINT',
'constraint' => 20,
'unsigned' => true,
],
'date' => [
'type' => 'date',
],
'episode_id' => [
'type' => 'BIGINT',
'constraint' => 20,
'unsigned' => true,
],
'country_code' => [
'type' => 'VARCHAR',
'constraint' => 3,
'comment' => 'ISO 3166-1 code.',
],
'date' => [
'type' => 'date',
'age' => [
'type' => 'INT',
'constraint' => 10,
'unsigned' => true,
],
'hits' => [
'type' => 'INT',
......@@ -47,13 +41,7 @@ class AddAnalyticsEpisodesByCountry extends Migration
'default' => 1,
],
]);
$this->forge->addKey('id', true);
$this->forge->addUniqueKey([
'podcast_id',
'episode_id',
'country_code',
'date',
]);
$this->forge->addPrimaryKey(['podcast_id', 'episode_id', 'date']);
$this->forge->addField(
'`created_at` timestamp NOT NULL DEFAULT current_timestamp()'
);
......@@ -62,11 +50,11 @@ class AddAnalyticsEpisodesByCountry extends Migration
);
$this->forge->addForeignKey('podcast_id', 'podcasts', 'id');
$this->forge->addForeignKey('episode_id', 'episodes', 'id');
$this->forge->createTable('analytics_episodes_by_country');
$this->forge->createTable('analytics_podcasts_by_episode');
}
public function down()
{
$this->forge->dropTable('analytics_episodes_by_country');
$this->forge->dropTable('analytics_podcasts_by_episode');
}
}
......@@ -17,32 +17,45 @@ class AddAnalyticsPodcastsByPlayer extends Migration
public function up()
{
$this->forge->addField([
'id' => [
'type' => 'BIGINT',
'constraint' => 20,
'unsigned' => true,
'auto_increment' => true,
],
'podcast_id' => [
'type' => 'BIGINT',
'constraint' => 20,
'unsigned' => true,
],
'player' => [
'type' => 'VARCHAR',
'constraint' => 191,
],
'date' => [
'type' => 'date',
],
'app' => [
'type' => 'VARCHAR',
'constraint' => 128,
],
'device' => [
'type' => 'VARCHAR',
'constraint' => 32,
],
'os' => [
'type' => 'VARCHAR',
'constraint' => 32,
],
'bot' => [
'type' => 'TINYINT',
'constraint' => 1,
'default' => 0,
],
'hits' => [
'type' => 'INT',
'constraint' => 10,
'default' => 1,
],
]);
$this->forge->addKey('id', true);
$this->forge->addUniqueKey(['podcast_id', 'player', 'date']);
$this->forge->addPrimaryKey([
'podcast_id',
'app',
'device',
'os',
'bot',
'date',
]);
$this->forge->addField(
'`created_at` timestamp NOT NULL DEFAULT current_timestamp()'
);
......
......@@ -17,33 +17,26 @@ class AddAnalyticsPodcastsByCountry extends Migration
public function up()
{
$this->forge->addField([
'id' => [
'type' => 'BIGINT',
'constraint' => 20,
'unsigned' => true,
'auto_increment' => true,
],
'podcast_id' => [
'type' => 'BIGINT',
'constraint' => 20,
'unsigned' => true,
],
'date' => [
'type' => 'date',
],
'country_code' => [
'type' => 'VARCHAR',
'constraint' => 3,
'comment' => 'ISO 3166-1 code.',
],
'date' => [
'type' => 'date',
],
'hits' => [
'type' => 'INT',
'constraint' => 10,
'default' => 1,
],
]);
$this->forge->addKey('id', true);
$this->forge->addUniqueKey(['podcast_id', 'country_code', 'date']);
$this->forge->addPrimaryKey(['podcast_id', 'country_code', 'date']);
$this->forge->addField(
'`created_at` timestamp NOT NULL DEFAULT current_timestamp()'
);
......
<?php
/**
* Class AddAnalyticsWebsiteByCountry
* Creates analytics_website_by_country table in database
* Class AddAnalyticsPodcastsByRegion
* Creates analytics_podcasts_by_region table in database
* @copyright 2020 Podlibre
* @license https://www.gnu.org/licenses/agpl-3.0.en.html AGPL3
* @link https://castopod.org/
......@@ -12,29 +12,36 @@ namespace App\Database\Migrations;
use CodeIgniter\Database\Migration;
class AddAnalyticsWebsiteByCountry extends Migration
class AddAnalyticsPodcastsByRegion extends Migration
{
public function up()
{
$this->forge->addField([
'id' => [
'type' => 'BIGINT',
'constraint' => 20,
'unsigned' => true,
'auto_increment' => true,
],
'podcast_id' => [
'type' => 'BIGINT',
'constraint' => 20,
'unsigned' => true,
],
'date' => [
'type' => 'date',
],
'country_code' => [
'type' => 'VARCHAR',
'constraint' => 3,
'comment' => 'ISO 3166-1 code.',
],
'date' => [
'type' => 'date',
'region_code' => [
'type' => 'VARCHAR',
'constraint' => 3,
'comment' => 'ISO 3166-2 code.',
],
'latitude' => [
'type' => 'FLOAT',
'null' => true,
],
'longitude' => [
'type' => 'FLOAT',
'null' => true,
],
'hits' => [
'type' => 'INT',
......@@ -42,8 +49,12 @@ class AddAnalyticsWebsiteByCountry extends Migration
'default' => 1,
],
]);
$this->forge->addKey('id', true);
$this->forge->addUniqueKey(['podcast_id', 'country_code', 'date']);
$this->forge->addPrimaryKey([
'podcast_id',
'country_code',
'region_code',
'date',
]);
$this->forge->addField(
'`created_at` timestamp NOT NULL DEFAULT current_timestamp()'
);
......@@ -51,11 +62,11 @@ class AddAnalyticsWebsiteByCountry extends Migration
'`updated_at` timestamp NOT NULL DEFAULT current_timestamp() ON UPDATE current_timestamp()'
);
$this->forge->addForeignKey('podcast_id', 'podcasts', 'id');
$this->forge->createTable('analytics_website_by_country');
$this->forge->createTable('analytics_podcasts_by_region');
}
public function down()
{
$this->forge->dropTable('analytics_website_by_country');
$this->forge->dropTable('analytics_podcasts_by_region');
}
}
......@@ -17,32 +17,26 @@ class AddAnalyticsWebsiteByBrowser extends Migration
public function up()
{
$this->forge->addField([
'id' => [
'type' => 'BIGINT',