...
  View open merge request
Commits (7)
<?php
declare(strict_types = 1);
namespace T3o\TerFe2\Command;
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
use T3o\TerFe2\Domain\Model\Extension;
use T3o\TerFe2\Domain\Repository\ExtensionRepository;
use T3o\TerFe2\Domain\Repository\VersionRepository;
use T3o\TerFe2\Utility\VersionUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Mvc\Controller\CommandController;
use TYPO3\CMS\Extbase\Persistence\Generic\Typo3QuerySettings;
class PackagistCommandController extends CommandController
{
/**
* @var ExtensionRepository
*/
protected $extensionRepository;
/**
* @var VersionRepository
*/
protected $versionRepository;
/**
* @param ExtensionRepository $extensionRepository
*/
public function injectExtensionRepository(ExtensionRepository $extensionRepository)
{
$this->extensionRepository = $extensionRepository;
}
public function injectVersionRepository(VersionRepository $versionRepository)
{
$this->versionRepository = $versionRepository;
}
/**
* Fetch download data from Packagist
*
* Examples:
*
* --- Fetch download data from packagist
* $ typo3cms packagist:fetchdownloaddata 10
*
* @param int $limit You can set a limit, default is 10
*
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\IllegalObjectTypeException
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\InvalidQueryException
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\UnknownObjectException
*/
public function fetchDownloadDataCommand(int $limit = 10)
{
$querySettings = GeneralUtility::makeInstance(Typo3QuerySettings::class);
$querySettings->setRespectStoragePage(false);
$this->extensionRepository->setDefaultQuerySettings($querySettings);
$this->versionRepository->setDefaultQuerySettings($querySettings);
$extensions = $this->extensionRepository->findWithComposerName($limit);
// User-Agent https://twitter.com/seldaek/status/1095420243315511297
$options = ['http' => ['user_agent' => 'Thanks a bunch! Fetching packagist Download Data for https://extensions.typo3.org - https://typo3.org/community/teams/typo3org/']];
$context = stream_context_create($options);
/** @var Extension $extension */
foreach ($extensions as $extension) {
$this->outputLine('<info> Processing: ' . $extension->getExtKey() . '</info>');
$packagistUrl = 'https://packagist.org/packages/' . $extension->getComposerName() . '/downloads.json';
/** @var Extension $extension */
$extension = $this->extensionRepository->findOneByExtKey($extension->getExtKey());
$downloadData = json_decode(file_get_contents($packagistUrl, false, $context), true);
foreach ($downloadData['package']['downloads']['versions'] as $version => $downloads) {
$versionFound = $this->versionRepository->findOneByExtensionAndVersionString($extension, $version);
if (null === $versionFound) {
continue;
}
if (!$version) {
continue;
}
VersionUtility::updateVersionCounter(
$extension->getExtKey(),
$versionFound->getUid(),
VersionUtility::DOWNLOAD_SOURCE_PACKAGIST,
$downloads['monthly']
);
}
}
}
}
......@@ -14,6 +14,7 @@ namespace T3o\TerFe2\Controller;
* The TYPO3 project - inspiring people to share!
*/
use T3o\TerFe2\Utility\VersionUtility;
use T3o\TerFe2\Validation\Validator\ComposerNameValidator;
/**
......@@ -410,12 +411,8 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
if (!empty($this->settings['countDownloads'])) {
$extensionKey = $extension->getExtKey();
$downloads = $this->session->get('downloads');
$versionRepository = $this->objectManager->get(\T3o\TerFe2\Domain\Repository\VersionRepository::class);
if (empty($downloads) || !in_array($extensionKey, $downloads)) {
// Add +1 to download counter and save immediately
$version->incrementDownloadCounter();
$versionRepository->update($version);
$this->persistenceManager->persistAll();
VersionUtility::updateVersionCounter($extensionKey, $version->getUid(), VersionUtility::DOWNLOAD_SOURCE_TER);
// Add extension key to session
$downloads[] = $extensionKey;
......
<?php
declare(strict_types = 1);
namespace T3o\TerFe2\Domain\Model;
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
use TYPO3\CMS\Extbase\DomainObject\AbstractEntity;
class Download extends AbstractEntity
{
/**
* @var string
*/
protected $extensionKey = '';
/**
* @var int
*/
protected $versionId = 0;
/**
* @var int
*/
protected $month = 0;
/**
* @var int
*/
protected $counter = 0;
/**
* @var int
*/
protected $source = 0;
/**
* @return string
*/
public function getExtensionKey(): string
{
return $this->extensionKey;
}
/**
* @param string $extensionKey
*/
public function setExtensionKey(string $extensionKey)
{
$this->extensionKey = $extensionKey;
}
/**
* @return int
*/
public function getVersionId(): int
{
return $this->versionId;
}
/**
* @param int $versionId
*/
public function setVersionId(int $versionId)
{
$this->versionId = $versionId;
}
/**
* @return int
*/
public function getMonth(): int
{
return $this->month;
}
/**
* @param int $month
*/
public function setMonth(int $month)
{
$this->month = $month;
}
/**
* @return int
*/
public function getCounter(): int
{
return $this->counter;
}
/**
* @param int $counter
*/
public function setCounter(int $counter)
{
$this->counter = $counter;
}
/**
* @return int
*/
public function getSource(): int
{
return $this->source;
}
/**
* @param int $source
*/
public function setSource(int $source)
{
$this->source = $source;
}
}
<?php
namespace T3o\TerFe2\Domain\Model;
/*
......@@ -14,6 +15,7 @@ namespace T3o\TerFe2\Domain\Model;
* The TYPO3 project - inspiring people to share!
*/
use T3o\TerFe2\Domain\Repository\DownloadRepository;
use TYPO3\CMS\Extbase\Persistence\ObjectStorage;
/**
......@@ -121,6 +123,21 @@ class Extension extends \TYPO3\CMS\Extbase\DomainObject\AbstractEntity
*/
protected $likes = 0;
/**
* @var int
*/
protected $lastDownloadSync;
/**
* @var DownloadRepository
*/
protected $downloadRepository;
public function injectDownloadRepository(DownloadRepository $downloadRepository)
{
$this->downloadRepository = $downloadRepository;
}
/**
* Initialize all ObjectStorage instances.
*/
......@@ -540,6 +557,30 @@ class Extension extends \TYPO3\CMS\Extbase\DomainObject\AbstractEntity
return json_encode($versions);
}
/**
* @return string
*/
public function getDownloadsByTimeIntervalAsJson(): string
{
if (empty($this->versions)) {
return '';
}
$intervals = [];
// The for loop is turned around to ensure the sorting order in the multi dimensional array.
for ($i = 5; $i >= 0; $i--) {
$time = strtotime('-' . $i . 'month');
$month = date('Ym', $time);
$downloads = $this->downloadRepository->findDownloadCounterByMonthAndExtensionKeyTotal($month, $this->getExtKey());
$intervals['release'][] = $month;
$intervals['downloads'][] = $downloads;
$intervals['versions'][] = '';
}
return json_encode($intervals);
}
/**
* @return \DateTime|null
*/
......@@ -600,4 +641,20 @@ class Extension extends \TYPO3\CMS\Extbase\DomainObject\AbstractEntity
$this->likes--;
}
}
/**
* @return int
*/
public function getLastDownloadSync(): int
{
return $this->lastDownloadSync;
}
/**
* @param int $lastDownloadSync
*/
public function setLastDownloadSync(int $lastDownloadSync)
{
$this->lastDownloadSync = $lastDownloadSync;
}
}
<?php
declare(strict_types = 1);
namespace T3o\TerFe2\Domain\Repository;
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
use T3o\TerFe2\Domain\Model\Download;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
use TYPO3\CMS\Core\Utility\ClassNamingUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Object\ObjectManager;
use TYPO3\CMS\Extbase\Persistence\Generic\Typo3QuerySettings;
class DownloadRepository extends AbstractRepository
{
/**
* @var string
*/
protected $tableName = 'tx_terfe2_domain_model_download';
/**
* @var QueryBuilder
*/
protected $queryBuilder = QueryBuilder::class;
/**
* QueueRepository constructor.
*/
public function __construct()
{
$this->queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($this->tableName);
$this->objectManager = GeneralUtility::makeInstance(ObjectManager::class);
$this->objectType = ClassNamingUtility::translateRepositoryNameToModelName($this->getRepositoryClassName());
}
/**
* @param $month
* @param $extensionKey
* @param $versionId
* @param $source
*
* @return object|null
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\InvalidQueryException
*/
public function findDownloadCounterByMonthAndExtensionKey($month, $extensionKey, $versionId, $source)
{
$query = $this->createQuery();
$query->matching(
$query->logicalAnd(
[
$query->equals('month', $month),
$query->equals('extensionKey', $extensionKey),
$query->equals('versionId', $versionId),
$query->equals('source', $source),
]
)
);
return $query->execute()->getFirst();
}
/**
* @param $month
* @param $extensionKey
*
* @return int
*/
public function findDownloadCounterByMonthAndExtensionKeyTotal($month, $extensionKey)
{
$total = 0;
$querySettings = GeneralUtility::makeInstance(Typo3QuerySettings::class);
$querySettings->setRespectStoragePage(false);
$query = $this->createQuery();
$query->setQuerySettings($querySettings);
$query->matching(
$query->logicalAnd(
[
$query->equals('month', $month),
$query->equals('extensionKey', $extensionKey)
]
)
);
$downloads = $query->execute()->toArray();
/** @var Download $download */
foreach ($downloads as $download) {
$total = $total + $download->getCounter();
}
return $total;
}
}
......@@ -14,6 +14,10 @@ namespace T3o\TerFe2\Domain\Repository;
* The TYPO3 project - inspiring people to share!
*/
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Persistence\Generic\Typo3QuerySettings;
use TYPO3\CMS\Extbase\Persistence\QueryInterface;
/**
* Repository for \T3o\TerFe2\Domain\Model\Extension
*/
......@@ -188,4 +192,26 @@ class ExtensionRepository extends \T3o\TerFe2\Domain\Repository\AbstractReposito
return $query->execute();
}
/**
* @param int $limit
* @return array|\TYPO3\CMS\Extbase\Persistence\QueryResultInterface
*/
public function findWithComposerName(int $limit = 10)
{
$querySettings = GeneralUtility::makeInstance(Typo3QuerySettings::class);
$querySettings->setRespectStoragePage(false);
$query = $this->createQuery();
$query->setQuerySettings($querySettings);
$query->matching(
$query->logicalNot(
$query->equals('composer_name', '')
)
);
$query->setOrderings(['lastDownloadSync' => QueryInterface::ORDER_ASCENDING]);
$query->setLimit($limit);
return $query->execute();
}
}
<?php
namespace T3o\TerFe2\Utility;
/*
......@@ -14,11 +15,21 @@ namespace T3o\TerFe2\Utility;
* The TYPO3 project - inspiring people to share!
*/
use T3o\TerFe2\Domain\Model\Download;
use T3o\TerFe2\Domain\Model\Extension;
use T3o\TerFe2\Domain\Repository\DownloadRepository;
use T3o\TerFe2\Domain\Repository\ExtensionRepository;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Object\ObjectManager;
use TYPO3\CMS\Extbase\Persistence\Generic\PersistenceManager;
/**
* Utilities to manage versions
*/
class VersionUtility
{
const DOWNLOAD_SOURCE_TER = 1;
const DOWNLOAD_SOURCE_PACKAGIST = 2;
/**
* Build version from integer
......@@ -36,7 +47,7 @@ class VersionUtility
$parts = [
substr($versionString, 0, 3),
substr($versionString, 3, 3),
substr($versionString, 6, 3)
substr($versionString, 6, 3),
];
return intval($parts[0]) . '.' . intval($parts[1]) . '.' . intval($parts[2]);
}
......@@ -65,4 +76,48 @@ class VersionUtility
return false;
}
/**
* @param string $extensionKey
* @param int $versionId
* @param int $source
* @param int $counter
*
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\IllegalObjectTypeException
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\InvalidQueryException
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\UnknownObjectException
*/
public static function updateVersionCounter(String $extensionKey, int $versionId, int $source = self::DOWNLOAD_SOURCE_TER, int $counter = 1)
{
$objectManager = GeneralUtility::makeInstance(ObjectManager::class);
$persistenceManager = $objectManager->get(PersistenceManager::class);
$extensionRepository = $objectManager->get(ExtensionRepository::class);
$month = date('Ym');
$downloadRepository = $objectManager->get(DownloadRepository::class);
/** @var Download $downloadInfo */
$downloadInfo = $downloadRepository->findDownloadCounterByMonthAndExtensionKey($month, $extensionKey, $versionId, $source);
if (null === $downloadInfo) {
$downloadInfo = new Download();
$downloadInfo->setExtensionKey($extensionKey);
$downloadInfo->setSource($source);
$downloadInfo->setMonth($month);
$downloadInfo->setVersionId($versionId);
$downloadInfo->setCounter($counter);
$downloadRepository->add($downloadInfo);
} elseif ($downloadInfo->getCounter() > 0) {
$downloadInfo->setCounter($downloadInfo->getCounter() + 1);
$downloadRepository->update($downloadInfo);
}
// Sets Extension Last Download Sync to secure oldest syncs happens first.
/** @var Extension $extension */
$extension = $extensionRepository->findOneByExtKey($extensionKey);
$extension->setLastDownloadSync(time());
$extensionRepository->update($extension);
$persistenceManager->persistAll();
}
}
<?php
return [
'ctrl' => [
'title' => 'domain_model_download',
'label' => 'extension_key',
'tstamp' => 'tstamp',
'iconfile' => \TYPO3\CMS\Core\Utility\ExtensionManagementUtility::extRelPath('ter_fe2') . 'Resources/Public/Icons/download.png',
],
'interface' => [
'showRecordFieldList' => 'extension_key,version_id',
],
'types' => [
'1' => ['showitem' => 'extension_key'],
],
'palettes' => [
'1' => ['showitem' => ''],
],
'columns' => [
'extension_key' => [
'label' => 'LLL:EXT:ter_fe2/Resources/Private/Language/locallang_db.xlf:tx_terfe2_domain_model_download.extension_key',
'config' => [
'type' => 'input',
],
],
'version_id' => [
'label' => 'LLL:EXT:ter_fe2/Resources/Private/Language/locallang_db.xlf:tx_terfe2_domain_model_download.version_id',
'config' => [
'type' => 'input',
],
],
'source' => [
'label' => 'LLL:EXT:ter_fe2/Resources/Private/Language/locallang_db.xlf:tx_terfe2_domain_model_download.download_source',
'config' => [
'type' => 'input',
'size' => 2,
],
],
'month' => [
'exclude' => 1,
'label' => 'LLL:EXT:ter_fe2/Resources/Private/Language/locallang_db.xlf:tx_terfe2_domain_model_download.month',
'config' => [
'type' => 'input',
'size' => 11,
],
],
'counter' => [
'exclude' => 1,
'label' => 'LLL:EXT:ter_fe2/Resources/Private/Language/locallang_db.xlf:tx_terfe2_domain_model_download.counter',
'config' => [
'type' => 'input',
'size' => 11,
],
],
],
];
......@@ -19,10 +19,10 @@ return [
'iconfile' => \TYPO3\CMS\Core\Utility\ExtensionManagementUtility::extRelPath('ter_fe2') . 'Resources/Public/Icons/extension.gif',
],
'interface' => [
'showRecordFieldList' => 'ext_key,forge_link,last_update,last_maintained,tags,versions,last_version,frontend_user,downloads,composer_name,repository_url,paypal_url,external_manual,expire',
'showRecordFieldList' => 'ext_key,forge_link,last_update,last_maintained,tags,versions,last_version,frontend_user,downloads,composer_name,repository_url,paypal_url,external_manual,expire,last_download_sync',
],
'types' => [
'1' => ['showitem' => 'ext_key,forge_link,last_update,last_maintained,tags,versions,last_version,frontend_user,downloads,composer_name,repository_url,paypal_url,external_manual,expire'],
'1' => ['showitem' => 'ext_key,forge_link,last_update,last_maintained,tags,versions,last_version,frontend_user,downloads,composer_name,repository_url,paypal_url,external_manual,expire,last_download_sync'],
],
'palettes' => [
'1' => ['showitem' => ''],
......@@ -234,6 +234,14 @@ return [
'foreign_table' => 'fe_users',
'MM' => 'tx_terfe2_extension_feuser_mm'
]
],
'last_download_sync' => [
'exclude' => 1,
'label' => 'LLL:EXT:ter_fe2/Resources/Private/Language/locallang_db.xlf:tx_terfe2_domain_model_extension.last_download_sync',
'config' => [
'type' => 'input',
'eval' => 'date'
],
]
],
];
<?xml version="1.0" encoding="utf-8" standalone="yes" ?>
<xliff version="1.0">
<file source-language="en" datatype="plaintext" original="messages" date="2017-10-04T19:53:29Z" product-name="sfeid">
<header/>
<body>
<trans-unit id="extension_key" xml:space="preserve">
<source>Extension Key</source>
</trans-unit>
<trans-unit id="version_id" xml:space="preserve">
<source>Version ID</source>
</trans-unit>
<trans-unit id="download_source" xml:space="preserve">
<source>Download Source - 1: TER, 2: Packagist</source>
</trans-unit>
<trans-unit id="month" xml:space="preserve">
<source>Year and month</source>
</trans-unit>
<trans-unit id="counter" xml:space="preserve">
<source>counter</source>
</trans-unit>
</body>
</file>
</xliff>
......@@ -54,6 +54,9 @@
<trans-unit id="tx_terfe2_domain_model_extension.expire" xml:space="preserve">
<source>Extension key expires at</source>
</trans-unit>
<trans-unit id="tx_terfe2_domain_model_extension.last_download_sync" xml:space="preserve">
<source>Last download sync with Packagist</source>
</trans-unit>
<trans-unit id="tx_terfe2_domain_model_tag" xml:space="preserve">
<source>Tag</source>
</trans-unit>
......@@ -351,6 +354,12 @@
<trans-unit id="tx_terfe2_task_checkforexpiredextensions.description" xml:space="preserve">
<source>Check for expired extensions, if extension is expired a mail to the owner is sent.</source>
</trans-unit>
<trans-unit id="tx_terfe2_task_fetchPackagistDownloadData.name" xml:space="preserve">
<source>[TER FE2] Fetch download data from Packagist</source>
</trans-unit>
<trans-unit id="tx_terfe2_task_fetchPackagistDownloadData.description" xml:space="preserve">
<source>Fetches download data from packagist for extensions with composer name configured</source>
</trans-unit>
<trans-unit id="tx_terfe2_provider_mirrorprovider.name" xml:space="preserve">
<source>Mirror Servers</source>
</trans-unit>
......
......@@ -126,8 +126,11 @@
<h3>
<f:translate key="downloads_by_version" />
</h3>
<p>
Downloads since <f:format.date format="%d. %b %Y">{extension.dateOfFirstUpload}</f:format.date> including version {extension.lastVersion.versionString}: <f:format.number decimals="0" thousandsSeparator="," decimalSeparator=".">{extension.downloads}</f:format.number>
</p>
<script type="text/javascript">
var versionChartData = {extension.downloadsByVersionsAsJson -> f:format.raw()};
var versionChartData = {extension.downloadsByTimeIntervalAsJson -> f:format.raw()};
</script>
<div id="versionChart">
......
......@@ -19,7 +19,8 @@
},
"license": "GPL-2.0+",
"require": {
"typo3/cms-core": ">= 8.7.0, <= 8.7.99"
"typo3/cms-core": ">= 8.7.0, <= 8.7.99",
"ext-json": "*"
},
"autoload": {
"psr-4": {
......
......@@ -96,3 +96,6 @@ $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['scheduler']['tasks'][\T3o\TerFe2\Task
];
$GLOBALS['TYPO3_CONF_VARS']['FE']['eID_include']['ter_fe2:extension'] = 'EXT:ter_fe2/Classes/Controller/Eid/ExtensionController.php';
$GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['extbase']['commandControllers']['tx_ter_fe2_packagist'] =
\T3o\TerFe2\Command\PackagistCommandController::class;
......@@ -20,6 +20,7 @@ CREATE TABLE tx_terfe2_domain_model_extension (
paypal_url varchar(255) DEFAULT '' NOT NULL,
expire int(11) unsigned default '0' NOT NULL,
likes int(11) unsigned default '0' NOT NULL,
last_download_sync int(11) unsigned DEFAULT '0' NOT NULL,
tstamp int(11) unsigned DEFAULT '0' NOT NULL,
crdate int(11) unsigned DEFAULT '0' NOT NULL,
......@@ -225,6 +226,21 @@ CREATE TABLE fe_users (
liked_extensions int(11) unsigned DEFAULT '0' NOT NULL
);
CREATE TABLE `tx_terfe2_domain_model_download` (
uid int(11) NOT NULL auto_increment,
pid int(11) DEFAULT '0' NOT NULL,
extension_key varchar(255) DEFAULT '' NOT NULL,
version_id int(10) unsigned DEFAULT '0' NOT NULL,
month int(11) unsigned DEFAULT '0' NOT NULL,
source int(2) unsigned DEFAULT '1' NOT NULL,
counter int(11) unsigned DEFAULT '0' NOT NULL,
PRIMARY KEY (uid),
KEY parent (pid),
KEY extension_key (extension_key)
);
# ======================================================================
# Table configuration for table "tx_terfe2_extension_tag_mm"
# ======================================================================
......@@ -246,4 +262,4 @@ CREATE TABLE `tx_terfe2_extension_feuser_mm` (
KEY `uid_local` (`uid_local`),
KEY `uid_foreign` (`uid_foreign`)
);
);
\ No newline at end of file