Commit 03e23a28 authored by Benjamin Bellamy's avatar Benjamin Bellamy 💬 Committed by Yassine Doghri
Browse files

feat: update analytics so to meet IABv2 requirements 

- https://iabtechlab.com/wp-content/uploads/2017/12/Podcast_Measurement_v2-Dec-20-2017.pdf
  - no IP address is ever stored on the server. Only aggregate data is stored in the dababase.
  - rolling 24-hour window
  - castopod does not do pre-load
  - IP Blacklisting https://github.com/client9/ipcat
  - user-agent Filtering https://github.com/opawg/user-agents
  - ignores 2 bytes range "Range: 0-1" (performed by official Apple iOS Podcast app)
  - in case of partial content, adds up all requests to check >1mn was downloaded
  - identifying Uniques is done with a combination of IP Address and User Agent
- add AMcharts
- add some graphs
- add regions to analytics
- add ipcat blacklist
- enhance useragents performances
- add filesize and header size in order to calculate 1mn downloads
- update publisher ID3 field
- update castopod icon
- add disclaimer and warning import form translation
- update docs/setup-development.md

closes #10
parent 5417be00

.gitlab-ci.yml

+3 −5
Original line number Diff line number Diff line
@@ -39,13 +39,11 @@ bundle_app: 
  script:

    # build all assets for views

    - npm run build

    # download GeoLite2-Country and opawg/user-agents archives and extract them to writable/uploads

    - wget -c "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key=$MAXMIND_LICENCE_KEY&suffix=tar.gz" -O - | tar -xz -C ./writable/uploads/

    - wget -c "https://github.com/opawg/user-agents/archive/master.tar.gz" -O - | tar -xz -C ./writable/uploads/

    # download GeoLite2-City archive and extract it to writable/uploads

    - wget -c "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-City&license_key=$MAXMIND_LICENCE_KEY&suffix=tar.gz" -O - | tar -xz -C ./writable/uploads/



    # rename extracted archives' folders

    - mv ./writable/uploads/GeoLite2-Country* ./writable/uploads/GeoLite2-Country

    - mv ./writable/uploads/user-agents* ./writable/uploads/user-agents

    - mv ./writable/uploads/GeoLite2-City* ./writable/uploads/GeoLite2-City



    # create bundle folder: uses .rsync-filter (-F) file to copy only needed files

    - rsync -avF --progress . ./bundle

DEPENDENCIES.md

+6 −2
Original line number Diff line number Diff line
@@ -12,16 +12,20 @@ PHP Dependencies: 
- [commonmark](https://commonmark.thephpleague.com/) ([BSD 3-Clause "New" or "Revised" License](https://github.com/thephpleague/commonmark/blob/latest/LICENSE))

- [phpdotenv](https://github.com/vlucas/phpdotenv) ([ BSD-3-Clause License ](https://github.com/vlucas/phpdotenv/blob/master/LICENSE))

- [HTML To Markdown for PHP](https://github.com/thephpleague/html-to-markdown) ([MIT License](https://github.com/thephpleague/html-to-markdown/blob/master/LICENSE))

- [podlibre/user-agents-php](https://github.com/podlibre/user-agents-php) ([MIT License](https://github.com/podlibre/user-agents-php/blob/main/LICENSE))

- [podlibre/ipcat](https://github.com/podlibre/ipcat) ([GNU General Public License v3.0](https://github.com/podlibre/ipcat/blob/master/LICENSE))



Javascript dependencies:



- [rollup](https://rollupjs.org/) ([MIT License](https://github.com/rollup/rollup/blob/master/LICENSE.md))

- [tailwindcss](https://tailwindcss.com/) ([MIT License](https://github.com/tailwindcss/tailwindcss/blob/master/LICENSE))

- [ProseMirror](https://prosemirror.net/) ([MIT License](https://github.com/ProseMirror/prosemirror/blob/master/LICENSE))

- [D3: Data-Driven Documents](https://d3js.org) ([BSD 3-Clause "New" or "Revised" License](https://github.com/d3/d3/blob/master/LICENSE))

- [amCharts 4](https://github.com/amcharts/amcharts4) ([Free amCharts license](https://github.com/amcharts/amcharts4/blob/master/dist/script/LICENSE))

- [Choices.js](https://joshuajohnson.co.uk/Choices/) ([MIT License](https://github.com/jshjohnson/Choices/blob/master/LICENSE))



Other:



- [RemixIcon](https://remixicon.com/) ([Apache License 2.0](https://github.com/Remix-Design/RemixIcon/blob/master/License))

- [User agent list](https://github.com/opawg/user-agents) ([by Open Podcast Analytics Working Group](https://github.com/opawg)) ([MIT license](https://github.com/opawg/user-agents/blob/master/LICENSE))
 
- [OPAWG/User agent list](https://github.com/opawg/user-agents) ([by Open Podcast Analytics Working Group](https://github.com/opawg)) ([MIT license](https://github.com/opawg/user-agents/blob/master/LICENSE))

- [client9/ipcat](https://github.com/client9/ipcat) ([GNU General Public License v3.0](https://github.com/client9/ipcat/blob/master/LICENSE))

- [GeoLite2 City](https://dev.maxmind.com/geoip/geoip2/geolite2/) ([Attribution-ShareAlike 4.0 International (CC BY-SA 4.0)](https://www.maxmind.com/en/geolite2/eula))

app/Config/Routes.php

+28 −4
Original line number Diff line number Diff line
@@ -53,10 +53,14 @@ $routes->group(config('App')->installGateway, function ($routes) { 
    ]);

});



// Route for podcast audio file analytics (/stats/podcast_id/episode_id/podcast_folder/filename.mp3)

$routes->add('stats/(:num)/(:num)/(:any)', 'Analytics::hit/$1/$2/$3', [

// Route for podcast audio file analytics (/audio/podcast_id/episode_id/bytes_threshold/filesize/podcast_folder/filename.mp3)

$routes->add(

    'audio/(:num)/(:num)/(:num)/(:num)/(:any)',

    'Analytics::hit/$1/$2/$3/$4/$5',

    [

        'as' => 'analytics_hit',

]);

    ]

);



// Show the Unknown UserAgents

$routes->get('.well-known/unknown-useragents', 'UnknownUserAgents');
@@ -113,6 +117,26 @@ $routes->group( 
                    'as' => 'podcast-delete',

                    'filter' => 'permission:podcasts-delete',

                ]);

                $routes->get('analytics', 'Podcast::analytics/$1', [

                    'as' => 'podcast-analytics',

                    'filter' => 'permission:podcasts-view,podcast-view',

                ]);

                $routes->get(

                    'analytics-data/(:segment)/(:segment)',

                    'AnalyticsData::getData/$1/$2/$3',

                    [

                        'as' => 'analytics-data',

                        'filter' => 'permission:podcasts-view,podcast-view',

                    ]

                );

                $routes->get(

                    'analytics-data/(:segment)/(:segment)/(:num)',

                    'AnalyticsData::getData/$1/$2/$3/$4',

                    [

                        'as' => 'analytics-filtered-data',

                        'filter' => 'permission:podcasts-view,podcast-view',

                    ]

                );



                // Podcast episodes

                $routes->group('episodes', function ($routes) {

app/Controllers/Admin/AnalyticsData.php

0 → 100644
+69 −0
Original line number Diff line number Diff line 
<?php



/**

 * @copyright  2020 Podlibre

 * @license    https://www.gnu.org/licenses/agpl-3.0.en.html AGPL3

 * @link       https://castopod.org/

 */



namespace App\Controllers\Admin;



use App\Models\PodcastModel;

use App\Models\EpisodeModel;



class AnalyticsData extends BaseController

{

    /**

     * @var \App\Entities\Podcast|null

     */

    protected $podcast;

    protected $className;

    protected $methodName;

    protected $episode;



    public function _remap($method, ...$params)

    {

        if (count($params) > 2) {

            if (!($this->podcast = (new PodcastModel())->find($params[0]))) {

                throw \CodeIgniter\Exceptions\PageNotFoundException::forPageNotFound(

                    'Podcast not found: ' . $params[0]

                );

            }

            $this->className = '\App\Models\Analytics' . $params[1] . 'Model';

            $this->methodName = 'getData' . $params[2];

            if (count($params) > 3) {

                if (

                    !($this->episode = (new EpisodeModel())

                        ->where([

                            'podcast_id' => $this->podcast->id,

                            'id' => $params[3],

                        ])

                        ->first())

                ) {

                    throw \CodeIgniter\Exceptions\PageNotFoundException::forPageNotFound(

                        'Episode not found: ' . $params[3]

                    );

                }

            }

        }



        return $this->$method();

    }

    public function getData()

    {

        $analytics_model = new $this->className();

        $methodName = $this->methodName;

        if ($this->episode) {

            return $this->response->setJSON(

                $analytics_model->$methodName(

                    $this->podcast->id,

                    $this->episode->id

                )

            );

        } else {

            return $this->response->setJSON(

                $analytics_model->$methodName($this->podcast->id)

            );

        }

    }

}

app/Controllers/Admin/Podcast.php

+20 −6
Original line number Diff line number Diff line
@@ -58,6 +58,14 @@ class Podcast extends BaseController 
        return view('admin/podcast/view', $data);

    }



    public function analytics()

    {

        $data = ['podcast' => $this->podcast];



        replace_breadcrumb_params([0 => $this->podcast->title]);

        return view('admin/podcast/analytics', $data);

    }



    public function create()

    {

        helper(['form', 'misc']);
@@ -204,7 +212,9 @@ class Podcast extends BaseController 
        $podcast = new \App\Entities\Podcast([

            'name' => $this->request->getPost('name'),

            'imported_feed_url' => $this->request->getPost('imported_feed_url'),



            'new_feed_url' => base_url(

                route_to('podcast_feed', $this->request->getPost('name'))

            ),

            'title' => $feed->channel[0]->title,

            'description' => $feed->channel[0]->description,

            'image' => download_file($nsItunes->image->attributes()),
@@ -214,7 +224,9 @@ class Podcast extends BaseController 
                ? null

                : (in_array($nsItunes->explicit, ['yes', 'true'])

                    ? 'explicit'

                    : null),

                    : (in_array($nsItunes->explicit, ['no', 'false'])

                        ? 'clean'

                        : null)),

            'owner_name' => $nsItunes->owner->name,

            'owner_email' => $nsItunes->owner->email,

            'publisher' => $nsItunes->author,
@@ -302,11 +314,13 @@ class Podcast extends BaseController 
                'image' => empty($nsItunes->image->attributes())

                    ? null

                    : download_file($nsItunes->image->attributes()),

                'explicit' => $nsItunes->explicit

                    ? (in_array($nsItunes->explicit, ['yes', 'true'])

                'parental_advisory' => empty($nsItunes->explicit)

                    ? null

                    : (in_array($nsItunes->explicit, ['yes', 'true'])

                        ? 'explicit'

                        : null)

                    : null,

                        : (in_array($nsItunes->explicit, ['no', 'false'])

                            ? 'clean'

                            : null)),

                'number' =>

                    $this->request->getPost('force_renumber') === 'yes'

                        ? $itemNumber