Commit 247ae182 authored by Yassine Doghri's avatar Yassine Doghri
Browse files

refactor(analytics): move all analytics files to a new Libraries/Analytics folder

- add page hit on podcast activity page
- update development docs
parent 1c0d6cee
......@@ -20,9 +20,9 @@ RUN docker-php-ext-configure gd --with-jpeg-dir=/usr/include/ \
RUN docker-php-ext-install mysqli && docker-php-ext-enable mysqli
RUN echo "file_uploads = On\n" \
"memory_limit = 100M\n" \
"upload_max_filesize = 100M\n" \
"post_max_size = 120M\n" \
"memory_limit = 512M\n" \
"upload_max_filesize = 500M\n" \
"post_max_size = 512M\n" \
"max_execution_time = 300\n" \
> /usr/local/etc/php/conf.d/uploads.ini
......
<?php
namespace Config;
use Analytics\Config\Analytics as AnalyticsBase;
class Analytics extends AnalyticsBase
{
/**
* --------------------------------------------------------------------
* Route filters options
* --------------------------------------------------------------------
*/
public $routeFilters = [
'analytics-full-data' => 'permission:podcasts-view,podcast-view',
'analytics-data' => 'permission:podcasts-view,podcast-view',
'analytics-filtered-data' => 'permission:podcasts-view,podcast-view',
];
public function __construct()
{
parent::__construct();
// set the analytics gateway behind the admin gateway.
// Only logged in users should be able to view analytics
$this->gateway = config('App')->adminGateway . '/analytics';
}
public function getEnclosureUrl($enclosureUri)
{
helper('media');
return media_base_url($enclosureUri);
}
}
......@@ -43,6 +43,7 @@ class Autoload extends AutoloadConfig
APP_NAMESPACE => APPPATH, // For custom app namespace
'Config' => APPPATH . 'Config',
'ActivityPub' => APPPATH . 'Libraries/ActivityPub',
'Analytics' => APPPATH . 'Libraries/Analytics',
];
/**
......
......@@ -70,18 +70,6 @@ $routes->group(config('App')->installGateway, function ($routes) {
]);
});
// Route for podcast audio file analytics (/audio/pack(podcast_id,episode_id,bytes_threshold,filesize,duration,date)/podcast_folder/filename.mp3)
$routes->head('audio/(:base64)/(:any)', 'Analytics::hit/$1/$2', [
'as' => 'analytics_hit',
]);
$routes->get('audio/(:base64)/(:any)', 'Analytics::hit/$1/$2', [
'as' => 'analytics_hit',
]);
// Show the Unknown UserAgents
$routes->get('.well-known/unknown-useragents', 'UnknownUserAgents');
$routes->get('.well-known/unknown-useragents/(:num)', 'UnknownUserAgents/$1');
$routes->get('.well-known/platforms', 'Platform');
// Admin area
......@@ -237,31 +225,6 @@ $routes->group(
);
});
$routes->get(
'analytics-data/(:segment)',
'AnalyticsData::getData/$1/$2',
[
'as' => 'analytics-full-data',
'filter' => 'permission:podcasts-view,podcast-view',
],
);
$routes->get(
'analytics-data/(:segment)/(:segment)',
'AnalyticsData::getData/$1/$2/$3',
[
'as' => 'analytics-data',
'filter' => 'permission:podcasts-view,podcast-view',
],
);
$routes->get(
'analytics-data/(:segment)/(:segment)/(:num)',
'AnalyticsData::getData/$1/$2/$3/$4',
[
'as' => 'analytics-filtered-data',
'filter' => 'permission:podcasts-view,podcast-view',
],
);
// Podcast episodes
$routes->group('episodes', function ($routes) {
$routes->get('/', 'Episode::list/$1', [
......
<?php
/**
* @copyright 2020 Podlibre
* @license https://www.gnu.org/licenses/agpl-3.0.en.html AGPL3
* @link https://castopod.org/
*/
namespace App\Controllers\Admin;
use App\Models\PodcastModel;
use App\Models\EpisodeModel;
class AnalyticsData extends BaseController
{
/**
* @var \App\Entities\Podcast|null
*/
protected $podcast;
protected $className;
protected $methodName;
protected $episode;
public function _remap($method, ...$params)
{
if (count($params) > 1) {
if (!($this->podcast = (new PodcastModel())->find($params[0]))) {
throw \CodeIgniter\Exceptions\PageNotFoundException::forPageNotFound(
'Podcast not found: ' . $params[0]
);
}
$this->className = '\App\Models\Analytics' . $params[1] . 'Model';
$this->methodName =
'getData' . (empty($params[2]) ? '' : $params[2]);
if (count($params) > 3) {
if (
!($this->episode = (new EpisodeModel())
->where([
'podcast_id' => $this->podcast->id,
'id' => $params[3],
])
->first())
) {
throw \CodeIgniter\Exceptions\PageNotFoundException::forPageNotFound(
'Episode not found: ' . $params[3]
);
}
}
}
return $this->$method();
}
public function getData()
{
$analytics_model = new $this->className();
$methodName = $this->methodName;
if ($this->episode) {
return $this->response->setJSON(
$analytics_model->$methodName(
$this->podcast->id,
$this->episode->id
)
);
} else {
return $this->response->setJSON(
$analytics_model->$methodName($this->podcast->id)
);
}
}
}
......@@ -44,8 +44,6 @@ class Episode extends BaseController
public function index()
{
$episodeModel = new EpisodeModel();
self::triggerWebpageHit($this->podcast->id);
$locale = service('request')->getLocale();
......@@ -65,7 +63,7 @@ class Episode extends BaseController
'persons' => $podcastPersons,
];
$secondsToNextUnpublishedEpisode = $episodeModel->getSecondsToNextUnpublishedEpisode(
$secondsToNextUnpublishedEpisode = (new EpisodeModel())->getSecondsToNextUnpublishedEpisode(
$this->podcast->id,
);
......@@ -112,7 +110,6 @@ class Episode extends BaseController
$cacheName = "page_podcast{$this->episode->podcast_id}_episode{$this->episode->id}_embeddable_player_{$theme}_{$locale}";
if (!($cachedView = cache($cacheName))) {
$episodeModel = new EpisodeModel();
$theme = EpisodeModel::$themes[$theme];
$data = [
......@@ -121,7 +118,7 @@ class Episode extends BaseController
'theme' => $theme,
];
$secondsToNextUnpublishedEpisode = $episodeModel->getSecondsToNextUnpublishedEpisode(
$secondsToNextUnpublishedEpisode = (new EpisodeModel())->getSecondsToNextUnpublishedEpisode(
$this->podcast->id,
);
......
......@@ -111,7 +111,7 @@ class Install extends Controller
// show database config view to fix value
session()->setFlashdata(
'error',
lang('Install.messages.databaseConnectError')
lang('Install.messages.databaseConnectError'),
);
return view('install/database_config');
......@@ -159,7 +159,7 @@ class Install extends Controller
return redirect()
->to(
(empty(host_url()) ? config('App')->baseURL : host_url()) .
config('App')->installGateway
config('App')->installGateway,
)
->withInput()
->with('errors', $this->validator->getErrors());
......@@ -181,8 +181,8 @@ class Install extends Controller
// redirect to full install url with new baseUrl input
return redirect(0)->to(
reduce_double_slashes(
$baseUrl . '/' . config('App')->installGateway
)
$baseUrl . '/' . config('App')->installGateway,
),
);
}
......@@ -209,14 +209,14 @@ class Install extends Controller
self::writeEnv([
'database.default.hostname' => $this->request->getPost(
'db_hostname'
'db_hostname',
),
'database.default.database' => $this->request->getPost('db_name'),
'database.default.username' => $this->request->getPost(
'db_username'
'db_username',
),
'database.default.password' => $this->request->getPost(
'db_password'
'db_password',
),
'database.default.DBPrefix' => $this->request->getPost('db_prefix'),
]);
......@@ -258,6 +258,7 @@ class Install extends Controller
!$migrations->setNamespace('Myth\Auth')->latest();
!$migrations->setNamespace('ActivityPub')->latest();
!$migrations->setNamespace('Analytics')->latest();
!$migrations->setNamespace(APP_NAMESPACE)->latest();
}
......@@ -296,7 +297,7 @@ class Install extends Controller
[
'email' => 'required|valid_email|is_unique[users.email]',
'password' => 'required|strong_password',
]
],
);
if (!$this->validate($rules)) {
......
......@@ -37,6 +37,8 @@ class Podcast extends BaseController
public function activity()
{
self::triggerWebpageHit($this->podcast->id);
helper('persons');
$persons = [];
construct_person_array($this->podcast->persons, $persons);
......
......@@ -10,6 +10,7 @@
*/
namespace App\Database\Seeds;
use App\Models\PodcastModel;
use App\Models\EpisodeModel;
......@@ -23,16 +24,16 @@ class FakePodcastsAnalyticsSeeder extends Seeder
$jsonUserAgents = json_decode(
file_get_contents(
'https://raw.githubusercontent.com/opawg/user-agents/master/src/user-agents.json'
'https://raw.githubusercontent.com/opawg/user-agents/master/src/user-agents.json',
),
true
true,
);
$jsonRSSUserAgents = json_decode(
file_get_contents(
'https://raw.githubusercontent.com/opawg/podcast-rss-useragents/master/src/rss-ua.json'
'https://raw.githubusercontent.com/opawg/podcast-rss-useragents/master/src/rss-ua.json',
),
true
true,
);
if ($podcast) {
......@@ -60,7 +61,7 @@ class FakePodcastsAnalyticsSeeder extends Seeder
->findAll();
foreach ($episodes as $episode) {
$age = floor(
($date - strtotime($episode->published_at)) / 86400
($date - strtotime($episode->published_at)) / 86400,
);
$proba1 = floor(exp(3 - $age / 40)) + 1;
......@@ -97,7 +98,7 @@ class FakePodcastsAnalyticsSeeder extends Seeder
$cityReader = new \GeoIp2\Database\Reader(
WRITEPATH .
'uploads/GeoLite2-City/GeoLite2-City.mmdb'
'uploads/GeoLite2-City/GeoLite2-City.mmdb',
);
$countryCode = 'N/A';
......@@ -196,7 +197,7 @@ class FakePodcastsAnalyticsSeeder extends Seeder
->insertBatch($analytics_podcasts_by_region);
}
} else {
echo "Create one podcast and some episodes first.\n";
echo "COULD NOT POPULATE DATABASE:\n\tCreate a podcast with episodes first.\n";
}
}
}
......@@ -10,6 +10,7 @@
*/
namespace App\Database\Seeds;
use App\Models\PodcastModel;
use App\Models\EpisodeModel;
......@@ -193,7 +194,7 @@ class FakeWebsiteAnalyticsSeeder extends Seeder
->findAll();
foreach ($episodes as $episode) {
$age = floor(
($date - strtotime($episode->published_at)) / 86400
($date - strtotime($episode->published_at)) / 86400,
);
$proba1 = floor(exp(3 - $age / 40)) + 1;
......@@ -254,7 +255,7 @@ class FakeWebsiteAnalyticsSeeder extends Seeder
->insertBatch($website_by_referer);
}
} else {
echo "Create one podcast and some episodes first.\n";
echo "COULD NOT POPULATE DATABASE:\n\tCreate a podcast with episodes first.\n";
}
}
}
......@@ -336,37 +336,14 @@ class Episode extends Entity
{
helper('analytics');
return base_url(
route_to(
'analytics_hit',
base64_url_encode(
pack(
'I*',
$this->attributes['podcast_id'],
$this->attributes['id'],
// bytes_threshold: number of bytes that must be downloaded for an episode to be counted in download analytics
// - if file is shorter than 60sec, then it's enclosure_filesize
// - if file is longer than 60 seconds then it's enclosure_headersize + 60 seconds
$this->attributes['enclosure_duration'] <= 60
? $this->attributes['enclosure_filesize']
: $this->attributes['enclosure_headersize'] +
floor(
(($this->attributes['enclosure_filesize'] -
$this->attributes[
'enclosure_headersize'
]) /
$this->attributes[
'enclosure_duration'
]) *
60,
),
$this->attributes['enclosure_filesize'],
$this->attributes['enclosure_duration'],
strtotime($this->attributes['published_at']),
),
),
$this->attributes['enclosure_uri'],
),
return generate_episode_analytics_url(
$this->podcast_id,
$this->id,
$this->enclosure_uri,
$this->enclosure_duration,
$this->enclosure_filesize,
$this->enclosure_headersize,
$this->published_at,
);
}
......@@ -520,9 +497,9 @@ class Episode extends Entity
empty($this->getPodcast()->partner_image_url)
? ''
: "<div><a href=\"{$this->getPartnerLink(
$serviceSlug
$serviceSlug,
)}\" rel=\"sponsored noopener noreferrer\" target=\"_blank\"><img src=\"{$this->getPartnerImage(
$serviceSlug
$serviceSlug,
)}\" alt=\"Partner image\" /></a></div>") .
$this->attributes['description_html'] .
(empty($this->getPodcast()->episode_description_footer_html)
......
<?php
/**
* @copyright 2020 Podlibre
* @license https://www.gnu.org/licenses/agpl-3.0.en.html AGPL3
* @link https://castopod.org/
*/
/**
* Encode Base64 for URLs
*/
function base64_url_encode($input)
{
return strtr(base64_encode($input), '+/=', '._-');
}
/**
* Decode Base64 from URL
*/
function base64_url_decode($input)
{
return base64_decode(strtr($input, '._-', '+/='));
}
/**
* Set user country in session variable, for analytics purpose
*/
function set_user_session_deny_list_ip()
{
$session = \Config\Services::session();
$session->start();
if (!$session->has('denyListIp')) {
$session->set(
'denyListIp',
\Podlibre\Ipcat\IpDb::find($_SERVER['REMOTE_ADDR']) != null,
);
}
}
/**
* Set user country in session variable, for analytics purpose
*/
function set_user_session_location()
{
$session = \Config\Services::session();
$session->start();
$location = [
'countryCode' => 'N/A',
'regionCode' => 'N/A',
'latitude' => null,
'longitude' => null,
];
// Finds location:
if (!$session->has('location')) {
try {
$cityReader = new \GeoIp2\Database\Reader(
WRITEPATH . 'uploads/GeoLite2-City/GeoLite2-City.mmdb',
);
$city = $cityReader->city($_SERVER['REMOTE_ADDR']);
$location = [
'countryCode' => empty($city->country->isoCode)
? 'N/A'
: $city->country->isoCode,
'regionCode' => empty($city->subdivisions[0]->isoCode)
? 'N/A'
: $city->subdivisions[0]->isoCode,
'latitude' => round($city->location->latitude, 3),
'longitude' => round($city->location->longitude, 3),
];
} catch (\Exception $e) {
// If things go wrong the show must go on and the user must be able to download the file
}
$session->set('location', $location);
}
}
/**
* Set user player in session variable, for analytics purpose
*/
function set_user_session_player()
{
$session = \Config\Services::session();
$session->start();
if (!$session->has('player')) {
$playerFound = null;
$userAgent = $_SERVER['HTTP_USER_AGENT'];
try {
$playerFound = \Opawg\UserAgentsPhp\UserAgents::find($userAgent);
} catch (\Exception $e) {
// If things go wrong the show must go on and the user must be able to download the file
}
if ($playerFound) {
$session->set('player', $playerFound);
} else {
$session->set('player', [
'app' => '- unknown -',
'device' => '',
'os' => '',
'bot' => 0,
]);
// Add to unknown list
try {
$db = \Config\Database::connect();
$procedureNameAnalyticsUnknownUseragents = $db->prefixTable(
'analytics_unknown_useragents',
);
$db->query("CALL $procedureNameAnalyticsUnknownUseragents(?)", [
$userAgent,
]);
} catch (\Exception $e) {
// If things go wrong the show must go on and the user must be able to download the file
}
}
}
}
/**
* Set user browser in session variable, for analytics purpose
*/
function set_user_session_browser()
{
$session = \Config\Services::session();
$session->start();
if (!$session->has('browser')) {
$browserName = '- Other -';
try {
$whichbrowser = new \WhichBrowser\Parser(getallheaders());
$browserName = $whichbrowser->browser->name;
} catch (\Exception $e) {
$browserName = '- Could not get browser name -';
}
if ($browserName == null) {
$browserName = '- Could not get browser name -';
}
$session->set('browser', $browserName);
}
}
/**
* Set user referer in session variable, for analytics purpose
*/
function set_user_session_referer()
{
$session = \Config\Services::session();
$session->start();
$newreferer = isset($_SERVER['HTTP_REFERER'])
? $_SERVER['HTTP_REFERER']
: '- Direct -';
$newreferer =
parse_url($newreferer, PHP_URL_HOST) ==
parse_url(current_url(false), PHP_URL_HOST)
? '- Direct -'
: $newreferer;
if (!$session->has('referer') or $newreferer != '- Direct -') {
$session->set('referer', $newreferer);
}
}
/**
* Set user entry page in session variable, for analytics purpose
*/
function set_user_session_entry_page()
{
$session = \Config\Services::session();
$session->start();
$entryPage = $_SERVER['REQUEST_URI'];
if (!$session->has('entryPage')) {
$session->set('entryPage', $entryPage);
}
}
function webpage_hit($podcast_id)
{
$session = \Config\Services::session();
$session->start();
if (!$session->get('denyListIp')) {
$db = \Config\Database::connect();
$referer = $session->get('referer');