diff --git a/.env.example b/.env.example index d897df2a..f6b419e2 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,6 @@ APP_ENV=local APP_DEBUG=true +DEV_EMAIL_TO=smarcet@gmail.com APP_KEY=SomeRandomString APP_URL=http://localhost APP_OAUTH_2_0_CLIENT_ID=clientid diff --git a/app/Console/Commands/SpammerProcess/.gitignore b/app/Console/Commands/SpammerProcess/.gitignore new file mode 100644 index 00000000..b32a23ab --- /dev/null +++ b/app/Console/Commands/SpammerProcess/.gitignore @@ -0,0 +1,4 @@ +env/ +.idea/ +__pycache__/ +user_classifier.pickle \ No newline at end of file diff --git a/app/Console/Commands/SpammerProcess/README.md b/app/Console/Commands/SpammerProcess/README.md new file mode 100644 index 00000000..b5718543 --- /dev/null +++ b/app/Console/Commands/SpammerProcess/README.md @@ -0,0 +1,19 @@ +## Dependencies + +````bas +$ sudo apt update +$ sudo apt install python3-pip python3-dev build-essential libssl-dev libffi-dev python3-setuptools python3-venv +libmysqlclient-dev +```` + + +## Virtual Env + +````bash +$ python3.6 -m venv env + +$ source env/bin/activate + +$ pip install -r requirements.txt + +```` \ No newline at end of file diff --git a/app/Console/Commands/SpammerProcess/RebuildUserSpammerEstimator.php b/app/Console/Commands/SpammerProcess/RebuildUserSpammerEstimator.php new file mode 100644 index 00000000..0c03a8b2 --- /dev/null +++ b/app/Console/Commands/SpammerProcess/RebuildUserSpammerEstimator.php @@ -0,0 +1,75 @@ +setTimeout(PHP_INT_MAX); + $process->setIdleTimeout(PHP_INT_MAX); + $process->run(); + + while ($process->isRunning()) { + } + + $output = $process->getOutput(); + + if (!$process->isSuccessful()) { + throw new Exception("Process Error!"); + } + } +} \ No newline at end of file diff --git a/app/Console/Commands/SpammerProcess/UserSpammerProcessor.php b/app/Console/Commands/SpammerProcess/UserSpammerProcessor.php new file mode 100644 index 00000000..7bb8a995 --- /dev/null +++ b/app/Console/Commands/SpammerProcess/UserSpammerProcessor.php @@ -0,0 +1,120 @@ +user_repository = $user_repository; + } + + /** + * @throws Exception + */ + public function handle() + { + $command = sprintf( + '%s/app/Console/Commands/SpammerProcess/estimator_process.sh "%s" "%s" "%s" "%s" "%s"', + base_path(), + base_path().'/app/Console/Commands/SpammerProcess', + env('DB_HOST','localhost'), + env('DB_USERNAME',''), + env('DB_PASSWORD',''), + env('DB_DATABASE','') + ); + $default = Config::get("database.default"); + $process = new Process($command); + $process->setTimeout(PHP_INT_MAX); + $process->setIdleTimeout(PHP_INT_MAX); + $process->run(); + + while ($process->isRunning()) { + } + + $csv_content = $process->getOutput(); + + if (!$process->isSuccessful()) { + throw new Exception("Process Error!"); + } + + $rows = CSVReader::load($csv_content); + + // send email with excerpt + + $users = []; + + foreach($rows as $row) { + $user_id = intval($row["ID"]); + $type = $row["Type"]; + $user = $this->user_repository->getById($user_id); + if(is_null($user) || !$user instanceof User) continue; + + $users[] = [ + 'id' => $user->getId(), + 'email' => $user->getEmail(), + 'full_name' => $user->getFullName(), + 'spam_type' => $type, + 'edit_link' => URL::route("edit_user", ["user_id" => $user->getId()], true) + ]; + } + + if(count($users) > 0){ + Mail::queue(new UserSpammerProcessorResultsEmail($users)); + } + } +} \ No newline at end of file diff --git a/app/Console/Commands/SpammerProcess/estimator_build.py b/app/Console/Commands/SpammerProcess/estimator_build.py new file mode 100644 index 00000000..09a1e51a --- /dev/null +++ b/app/Console/Commands/SpammerProcess/estimator_build.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# !/usr/bin/env python +# +# Copyright (c) 2020 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from openstack_member_spammer_estimator import EstimatorBuilder +import os + +# params +db_host = sys.argv[1] +db_user = sys.argv[2] +db_user_password = sys.argv[3] +db_name = sys.argv[4] +filename = 'user_classifier.pickle' +builder = EstimatorBuilder(filename=filename, db_host=db_host, db_user=db_user, db_user_password=db_user_password, + db_name=db_name) +script_dir = os.path.dirname(__file__) +pickle_file = os.path.join(script_dir, ) +if os.path.exists(pickle_file): + os.remove(pickle_file) + +builder.build() diff --git a/app/Console/Commands/SpammerProcess/estimator_build.sh b/app/Console/Commands/SpammerProcess/estimator_build.sh new file mode 100755 index 00000000..0ee04b5a --- /dev/null +++ b/app/Console/Commands/SpammerProcess/estimator_build.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (c) 2020 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +WORK_DIR=$1 +DB_HOST=$2 +DB_USER=$3 +DB_PASSWORD=$4 +DB_NAME=$5 + +export PYTHONPATH="$PYTHONPATH:$WORK_DIR"; + +cd $WORK_DIR; + +source env/bin/activate; + +python estimator_build.py $DB_HOST $DB_USER $DB_PASSWORD $DB_NAME; + +deactivate; \ No newline at end of file diff --git a/app/Console/Commands/SpammerProcess/estimator_process.py b/app/Console/Commands/SpammerProcess/estimator_process.py new file mode 100644 index 00000000..f2f7f696 --- /dev/null +++ b/app/Console/Commands/SpammerProcess/estimator_process.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python +# +# Copyright (c) 2020 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from openstack_member_spammer_estimator import EstimatorClassifier +import os + +# params +db_host = sys.argv[1] +db_user = sys.argv[2] +db_user_password = sys.argv[3] +db_name = sys.argv[4] +filename = 'user_classifier.pickle' + +classifier = EstimatorClassifier(db_host=db_host, db_user=db_user, db_user_password=db_user_password, db_name=db_name) +script_dir = os.path.dirname(__file__) +pickle_file = os.path.join(script_dir, filename) +if not os.path.exists(pickle_file): + raise Exception('File %s does not exists!' % pickle_file) + +res = classifier.classify(pickle_file) + +# output CSV file +print("ID,Type") +for row in res: + print("%s,%s" % row) diff --git a/app/Console/Commands/SpammerProcess/estimator_process.sh b/app/Console/Commands/SpammerProcess/estimator_process.sh new file mode 100755 index 00000000..0985b5fc --- /dev/null +++ b/app/Console/Commands/SpammerProcess/estimator_process.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (c) 2017 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +WORK_DIR=$1 +DB_HOST=$2 +DB_USER=$3 +DB_PASSWORD=$4 +DB_NAME=$5 + +export PYTHONPATH="$PYTHONPATH:$WORK_DIR"; + +cd $WORK_DIR; + +source env/bin/activate; + +python estimator_process.py $DB_HOST $DB_USER $DB_PASSWORD $DB_NAME; + +deactivate; \ No newline at end of file diff --git a/app/Console/Commands/SpammerProcess/requirements.txt b/app/Console/Commands/SpammerProcess/requirements.txt new file mode 100644 index 00000000..3a0717c8 --- /dev/null +++ b/app/Console/Commands/SpammerProcess/requirements.txt @@ -0,0 +1,26 @@ +openstack-member-spammer-estimator==1.0.2 +pkg-resources==0.0.0 +attrs==19.3.0 +configparser==4.0.2 +HTMLParser==0.0.2 +importlib-metadata==1.5.0 +joblib==0.14.1 +more-itertools==8.2.0 +mysqlclient==1.4.6 +nltk==3.4.5 +numpy==1.18.1 +packaging==20.3 +pandas==1.0.1 +pluggy==0.13.1 +py==1.8.1 +pyparsing==2.4.6 +pytest==5.3.5 +python-dateutil==2.8.1 +pytz==2019.3 +scikit-learn==0.22.2.post1 +scipy==1.4.1 +six==1.14.0 +sklearn==0.0 +wcwidth==0.1.8 +zipp==3.1.0 + diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index ad418498..3741930c 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -29,6 +29,8 @@ class Kernel extends ConsoleKernel Commands\CleanOAuth2StaleData::class, Commands\CleanOpenIdStaleData::class, Commands\CreateSuperAdmin::class, + Commands\SpammerProcess\RebuildUserSpammerEstimator::class, + Commands\SpammerProcess\UserSpammerProcessor::class, ]; /** @@ -41,5 +43,8 @@ class Kernel extends ConsoleKernel { $schedule->command('idp:oauth2-clean')->dailyAt("02:30")->withoutOverlapping(); $schedule->command('idp:openid-clean')->dailyAt("03:30")->withoutOverlapping(); + // user spammer + $schedule->command('user-spam:rebuild')->dailyAt("02:30")->withoutOverlapping(); + $schedule->command('user-spam:process')->dailyAt("03:30")->withoutOverlapping(); } } diff --git a/app/Events/UserActivated.php b/app/Events/UserActivated.php new file mode 100644 index 00000000..e09ce7b3 --- /dev/null +++ b/app/Events/UserActivated.php @@ -0,0 +1,19 @@ +user_id = $user_id; - $this->args = $args; - } - - /** - * @return int - */ - public function getUserId(): int - { - return $this->user_id; - } -} \ No newline at end of file +final class UserCreated extends UserEvent {} \ No newline at end of file diff --git a/app/Events/UserDeactivated.php b/app/Events/UserDeactivated.php new file mode 100644 index 00000000..ba986259 --- /dev/null +++ b/app/Events/UserDeactivated.php @@ -0,0 +1,19 @@ +user_id = $user_id; - } - - /** - * @return int - */ - public function getUserId(): int - { - return $this->user_id; - } -} \ No newline at end of file +class UserEmailUpdated extends UserEvent{} \ No newline at end of file diff --git a/app/Events/UserEmailVerified.php b/app/Events/UserEmailVerified.php index a45349f9..8d211095 100644 --- a/app/Events/UserEmailVerified.php +++ b/app/Events/UserEmailVerified.php @@ -11,35 +11,8 @@ * See the License for the specific language governing permissions and * limitations under the License. **/ -use Illuminate\Queue\SerializesModels; /** * Class UserEmailVerified * @package App\Events */ -final class UserEmailVerified -{ - use SerializesModels; - - /** - * @var int - */ - private $user_id; - - /** - * UserEmailVerified constructor. - * @param int $user_id - */ - public function __construct(int $user_id) - { - $this->user_id = $user_id; - } - - /** - * @return int - */ - public function getUserId(): int - { - return $this->user_id; - } - -} \ No newline at end of file +final class UserEmailVerified extends UserEvent {} \ No newline at end of file diff --git a/app/Events/UserEvent.php b/app/Events/UserEvent.php new file mode 100644 index 00000000..86985267 --- /dev/null +++ b/app/Events/UserEvent.php @@ -0,0 +1,44 @@ +user_id = $user_id; + } + + /** + * @return int + */ + public function getUserId(): int + { + return $this->user_id; + } +} \ No newline at end of file diff --git a/app/Events/UserLocked.php b/app/Events/UserLocked.php index ec5ba074..1fc7d53d 100644 --- a/app/Events/UserLocked.php +++ b/app/Events/UserLocked.php @@ -11,35 +11,8 @@ * See the License for the specific language governing permissions and * limitations under the License. **/ -use Illuminate\Queue\SerializesModels; - /** * Class UserLocked * @package App\Events */ -final class UserLocked -{ - use SerializesModels; - - /** - * @var int - */ - private $user_id; - - /** - * UserEmailVerified constructor. - * @param int $user_id - */ - public function __construct(int $user_id) - { - $this->user_id = $user_id; - } - - /** - * @return int - */ - public function getUserId(): int - { - return $this->user_id; - } -} \ No newline at end of file +final class UserLocked extends UserEvent{} \ No newline at end of file diff --git a/app/Events/UserPasswordResetRequestCreated.php b/app/Events/UserPasswordResetRequestCreated.php index 9f7f18eb..301a2bee 100644 --- a/app/Events/UserPasswordResetRequestCreated.php +++ b/app/Events/UserPasswordResetRequestCreated.php @@ -11,34 +11,8 @@ * See the License for the specific language governing permissions and * limitations under the License. **/ -use Illuminate\Queue\SerializesModels; /** * Class UserPasswordResetRequestCreated * @package App\Events */ -final class UserPasswordResetRequestCreated -{ - use SerializesModels; - - /** - * @var int - */ - private $id; - - /** - * UserEmailVerified constructor. - * @param int $user_id - */ - public function __construct(int $id) - { - $this->id = $id; - } - - /** - * @return int - */ - public function getId(): int - { - return $this->id; - } -} \ No newline at end of file +final class UserPasswordResetRequestCreated extends UserEvent{} \ No newline at end of file diff --git a/app/Events/UserPasswordResetSuccessful.php b/app/Events/UserPasswordResetSuccessful.php index 20554cbf..5052a2dd 100644 --- a/app/Events/UserPasswordResetSuccessful.php +++ b/app/Events/UserPasswordResetSuccessful.php @@ -11,35 +11,8 @@ * See the License for the specific language governing permissions and * limitations under the License. **/ -use Illuminate\Queue\SerializesModels; /** * Class UserPasswordResetSuccessful * @package App\Events */ -final class UserPasswordResetSuccessful -{ - - use SerializesModels; - - /** - * @var int - */ - private $user_id; - - /** - * UserEmailVerified constructor. - * @param int $user_id - */ - public function __construct(int $user_id) - { - $this->user_id = $user_id; - } - - /** - * @return int - */ - public function getUserId(): int - { - return $this->user_id; - } -} \ No newline at end of file +final class UserPasswordResetSuccessful extends UserEvent{} \ No newline at end of file diff --git a/app/Events/UserSpamStateUpdated.php b/app/Events/UserSpamStateUpdated.php new file mode 100644 index 00000000..8b5d2fa3 --- /dev/null +++ b/app/Events/UserSpamStateUpdated.php @@ -0,0 +1,22 @@ + 'App\Http\Controllers', 'middleware' => 'web' ], fu Route::group(array('prefix' => 'users'), function () { Route::get('', 'AdminController@listUsers'); Route::group(array('prefix' => '{user_id}'), function () { - Route::get('', 'AdminController@editUser'); + Route::get('', 'AdminController@editUser')->name("edit_user"); }); }); diff --git a/app/Mail/UserSpammerProcessorResultsEmail.php b/app/Mail/UserSpammerProcessorResultsEmail.php new file mode 100644 index 00000000..54f525c7 --- /dev/null +++ b/app/Mail/UserSpammerProcessorResultsEmail.php @@ -0,0 +1,52 @@ +users = $users; + } + + /** + * Build the message. + * + * @return $this + */ + public function build() + { + + $subject = sprintf("[%s] User Spammer Process Result", Config::get('app.app_name')); + + return $this->from(Config::get("mail.from")) + ->to(Config::get("mail.from")) + ->subject($subject) + ->view('emails.user_spammer_process_result'); + } + +} \ No newline at end of file diff --git a/app/ModelSerializers/Auth/UserSerializer.php b/app/ModelSerializers/Auth/UserSerializer.php index 37249a35..02db3bb2 100644 --- a/app/ModelSerializers/Auth/UserSerializer.php +++ b/app/ModelSerializers/Auth/UserSerializer.php @@ -31,6 +31,7 @@ final class PublicUserSerializer extends BaseUserSerializer { final class PrivateUserSerializer extends BaseUserSerializer { protected static $array_mappings = [ 'Email' => 'email:json_string', + 'SpamType' => 'spam_type:json_string', 'Identifier' => 'identifier:json_string', 'LastLoginDate' => 'last_login_date:datetime_epoch', 'Active' => 'active:json_boolean', diff --git a/app/Providers/EventServiceProvider.php b/app/Providers/EventServiceProvider.php index 09f9a448..44588557 100644 --- a/app/Providers/EventServiceProvider.php +++ b/app/Providers/EventServiceProvider.php @@ -12,10 +12,13 @@ * limitations under the License. **/ use App\Events\OAuth2ClientLocked; +use App\Events\UserActivated; +use App\Events\UserDeactivated; use App\Events\UserEmailUpdated; use App\Events\UserLocked; use App\Events\UserPasswordResetRequestCreated; use App\Events\UserPasswordResetSuccessful; +use App\Events\UserSpamStateUpdated; use App\libs\Auth\Repositories\IUserPasswordResetRequestRepository; use App\Mail\UserLockedEmail; use App\Mail\UserPasswordResetMail; @@ -81,6 +84,16 @@ final class EventServiceProvider extends ServiceProvider $user_service->sendVerificationEmail($user); }); + Event::listen(UserSpamStateUpdated::class, function($event) + { + $repository = App::make(IUserRepository::class); + $user = $repository->getById($event->getUserId()); + if(is_null($user)) return; + if(! $user instanceof User) return; + $user_service = App::make(IUserService::class); + $user_service->recalculateUserSpamType($user); + }); + Event::listen(UserEmailUpdated::class, function($event) { $repository = App::make(IUserRepository::class); @@ -93,7 +106,7 @@ final class EventServiceProvider extends ServiceProvider Event::listen(UserPasswordResetRequestCreated::class, function($event){ $repository = App::make(IUserPasswordResetRequestRepository::class); - $request = $repository->find($event->getId()); + $request = $repository->find($event->getUserId()); if(is_null($request)) return; }); @@ -123,5 +136,14 @@ final class EventServiceProvider extends ServiceProvider if(!$client instanceof Client) return; Mail::queue(new \App\Mail\OAuth2ClientLocked($client)); }); + + Event::listen(\Illuminate\Mail\Events\MessageSending::class, function($event){ + $devEmail = env('DEV_EMAIL_TO', null); + if(in_array(App::environment(), ['local','dev','testing']) && !empty($devEmail)){ + $event->message->setTo(explode(",", $devEmail)); + } + return true; + }); + } } diff --git a/app/Repositories/DoctrineSpamEstimatorFeedRepository.php b/app/Repositories/DoctrineSpamEstimatorFeedRepository.php new file mode 100644 index 00000000..27721f1d --- /dev/null +++ b/app/Repositories/DoctrineSpamEstimatorFeedRepository.php @@ -0,0 +1,46 @@ +getEntityManager()->createQueryBuilder(); + $qb->delete(SpamEstimatorFeed::class, 'e'); + $qb->where('e.email = :email'); + $qb->setParameter('email', trim($email)); + $qb->getQuery()->execute(); + } + catch(\Exception $ex){ + Log::error($ex); + } + } +} \ No newline at end of file diff --git a/app/Repositories/RepositoriesProvider.php b/app/Repositories/RepositoriesProvider.php index 8a235654..720f5a69 100644 --- a/app/Repositories/RepositoriesProvider.php +++ b/app/Repositories/RepositoriesProvider.php @@ -11,10 +11,11 @@ * See the License for the specific language governing permissions and * limitations under the License. **/ - +use App\libs\Auth\Models\SpamEstimatorFeed; use App\libs\Auth\Models\UserRegistrationRequest; use App\libs\Auth\Repositories\IBannedIPRepository; use App\libs\Auth\Repositories\IGroupRepository; +use App\libs\Auth\Repositories\ISpamEstimatorFeedRepository; use App\libs\Auth\Repositories\IUserExceptionTrailRepository; use App\libs\Auth\Repositories\IUserPasswordResetRequestRepository; use App\libs\Auth\Repositories\IUserRegistrationRequestRepository; @@ -216,6 +217,13 @@ final class RepositoriesProvider extends ServiceProvider } ); + App::singleton( + ISpamEstimatorFeedRepository::class, + function(){ + return EntityManager::getRepository(SpamEstimatorFeed::class); + } + ); + } public function provides() @@ -237,6 +245,7 @@ final class RepositoriesProvider extends ServiceProvider IResourceServerRepository::class, IWhiteListedIPRepository::class, IUserRegistrationRequestRepository::class, + ISpamEstimatorFeedRepository::class, ]; } } \ No newline at end of file diff --git a/app/Services/Auth/IUserService.php b/app/Services/Auth/IUserService.php index 0b04909e..666bd053 100644 --- a/app/Services/Auth/IUserService.php +++ b/app/Services/Auth/IUserService.php @@ -92,4 +92,10 @@ interface IUserService * @throws EntityNotFoundException */ public function createRegistrationRequest(string $client_id, array $payload):UserRegistrationRequest; + + /** + * @param User $user + * @return void + */ + public function recalculateUserSpamType(User $user):void; } \ No newline at end of file diff --git a/app/Services/Auth/UserService.php b/app/Services/Auth/UserService.php index a8eb2824..b448f4dd 100644 --- a/app/Services/Auth/UserService.php +++ b/app/Services/Auth/UserService.php @@ -14,8 +14,10 @@ use App\Events\UserPasswordResetSuccessful; use App\libs\Auth\Factories\UserFactory; use App\libs\Auth\Factories\UserRegistrationRequestFactory; +use App\libs\Auth\Models\SpamEstimatorFeed; use App\libs\Auth\Models\UserRegistrationRequest; use App\libs\Auth\Repositories\IGroupRepository; +use App\libs\Auth\Repositories\ISpamEstimatorFeedRepository; use App\libs\Auth\Repositories\IUserPasswordResetRequestRepository; use App\libs\Auth\Repositories\IUserRegistrationRequestRepository; use App\Mail\UserEmailVerificationRequest; @@ -69,6 +71,11 @@ final class UserService extends AbstractService implements IUserService */ private $client_repository; + /** + * @var ISpamEstimatorFeedRepository + */ + private $spam_estimator_feed_repository; + /** * UserService constructor. * @param IUserRepository $user_repository @@ -76,6 +83,7 @@ final class UserService extends AbstractService implements IUserService * @param IUserPasswordResetRequestRepository $request_reset_password_repository * @param IUserRegistrationRequestRepository $user_registration_request_repository * @param IClientRepository $client_repository + * @param ISpamEstimatorFeedRepository $spam_estimator_feed_repository * @param IUserNameGeneratorService $name_generator_service * @param ITransactionService $tx_service */ @@ -86,6 +94,7 @@ final class UserService extends AbstractService implements IUserService IUserPasswordResetRequestRepository $request_reset_password_repository, IUserRegistrationRequestRepository $user_registration_request_repository, IClientRepository $client_repository, + ISpamEstimatorFeedRepository $spam_estimator_feed_repository, IUserNameGeneratorService $name_generator_service, ITransactionService $tx_service ) @@ -96,6 +105,7 @@ final class UserService extends AbstractService implements IUserService $this->name_generator_service = $name_generator_service; $this->request_reset_password_repository = $request_reset_password_repository; $this->user_registration_request_repository = $user_registration_request_repository; + $this->spam_estimator_feed_repository = $spam_estimator_feed_repository; $this->client_repository = $client_repository; } @@ -369,4 +379,24 @@ final class UserService extends AbstractService implements IUserService return $request; }); } + + /** + * @inheritDoc + */ + public function recalculateUserSpamType(User $user): void + { + $this->tx_service->transaction(function() use($user) { + $this->spam_estimator_feed_repository->deleteByEmail($user->getEmail()); + switch($user->getSpamType()){ + case User::SpamTypeSpam: + $feed = SpamEstimatorFeed::buildFromUser($user, User::SpamTypeSpam); + $this->spam_estimator_feed_repository->add($feed); + break; + case User::SpamTypeHam: + $feed = SpamEstimatorFeed::buildFromUser($user, User::SpamTypeHam); + $this->spam_estimator_feed_repository->add($feed); + break; + } + }); + } } \ No newline at end of file diff --git a/app/Services/OpenId/UserService.php b/app/Services/OpenId/UserService.php index b77e57fc..c0dfff13 100644 --- a/app/Services/OpenId/UserService.php +++ b/app/Services/OpenId/UserService.php @@ -274,7 +274,6 @@ final class UserService extends AbstractService implements IUserService $user = $this->repository->getById($id); if(is_null($user) || !$user instanceof User) throw new EntityNotFoundException("user not found"); - $this->repository->delete($user); }); } diff --git a/app/libs/Auth/CustomAuthProvider.php b/app/libs/Auth/CustomAuthProvider.php index 2fe71f46..10f04af3 100644 --- a/app/libs/Auth/CustomAuthProvider.php +++ b/app/libs/Auth/CustomAuthProvider.php @@ -150,7 +150,7 @@ class CustomAuthProvider implements UserProvider //update user fields $user->setLastLoginDate(new \DateTime('now', new \DateTimeZone('UTC'))); $user->setLoginFailedAttempt(0); - $user->setActive(true); + $user->activate(); $user->clearResetPasswordRequests(); $auth_extensions = $this->auth_extension_service->getExtensions(); diff --git a/app/libs/Auth/Factories/UserFactory.php b/app/libs/Auth/Factories/UserFactory.php index 556316e6..c4b4b47e 100644 --- a/app/libs/Auth/Factories/UserFactory.php +++ b/app/libs/Auth/Factories/UserFactory.php @@ -134,8 +134,13 @@ final class UserFactory } } - if(isset($payload['active'])) - $user->setActive(boolval($payload['active'])); + if(isset($payload['active'])) { + $active = boolval($payload['active']); + if($active) + $user->activate(); + else + $user->deActivate(); + } if(isset($payload['public_profile_show_photo'])) $user->setPublicProfileShowPhoto(boolval($payload['public_profile_show_photo'])); diff --git a/app/libs/Auth/Models/SpamEstimatorFeed.php b/app/libs/Auth/Models/SpamEstimatorFeed.php new file mode 100644 index 00000000..eb828d49 --- /dev/null +++ b/app/libs/Auth/Models/SpamEstimatorFeed.php @@ -0,0 +1,149 @@ +first_name; + } + + /** + * @param string $first_name + */ + public function setFirstName(string $first_name): void + { + $this->first_name = $first_name; + } + + /** + * @return string + */ + public function getLastName(): ?string + { + return $this->last_name; + } + + /** + * @param string $last_name + */ + public function setLastName(string $last_name): void + { + $this->last_name = $last_name; + } + + /** + * @return string + */ + public function getEmail(): ?string + { + return $this->email; + } + + /** + * @param string $email + */ + public function setEmail(string $email): void + { + $this->email = $email; + } + + /** + * @return string + */ + public function getBio(): ?string + { + return $this->bio; + } + + /** + * @param string $bio + */ + public function setBio(string $bio): void + { + $this->bio = $bio; + } + + /** + * @return string + */ + public function getSpamType(): ?string + { + return $this->spam_type; + } + + /** + * @param string $spam_type + */ + public function setSpamType(string $spam_type): void + { + $this->spam_type = $spam_type; + } + + /** + * @param User $user + * @param string $spam_type + * @return SpamEstimatorFeed + */ + public static function buildFromUser(User $user, string $spam_type){ + $feed = new SpamEstimatorFeed; + $feed->spam_type = $spam_type; + $feed->email = $user->getEmail(); + $feed->first_name = $user->getFirstName(); + $feed->last_name = $user->getLastName(); + $feed->bio = $user->getBio(); + return $feed; + } +} \ No newline at end of file diff --git a/app/libs/Auth/Models/User.php b/app/libs/Auth/Models/User.php index 6928718d..46f178f7 100644 --- a/app/libs/Auth/Models/User.php +++ b/app/libs/Auth/Models/User.php @@ -11,10 +11,11 @@ * See the License for the specific language governing permissions and * limitations under the License. **/ - use App\Events\UserCreated; use App\Events\UserLocked; +use App\Events\UserSpamStateUpdated; use App\libs\Auth\Models\IGroupSlugs; +use Doctrine\ORM\Event\PreUpdateEventArgs; use Illuminate\Support\Facades\Event; use App\Events\UserEmailVerified; use Doctrine\Common\Collections\Criteria; @@ -48,6 +49,16 @@ class User extends BaseEntity use Authenticatable; use CanResetPasswordTrait; + const SpamTypeNone = 'None'; + const SpamTypeSpam = 'Spam'; + const SpamTypeHam = 'Ham'; + + const ValidSpamTypes = [ + self::SpamTypeNone, + self::SpamTypeSpam, + self::SpamTypeHam + ]; + /** * @ORM\Column(name="identifier", type="string") * @var string @@ -263,6 +274,12 @@ class User extends BaseEntity */ private $birthday; + /** + * @ORM\Column(name="spam_type", nullable=false, type="string") + * @var string + */ + private $spam_type; + // relations /** @@ -803,21 +820,12 @@ class User extends BaseEntity return $this->active; } - /** - * @param bool $active - */ - public function setActive(bool $active): void - { - $this->active = $active; - } - - /** * @return $this */ public function lock() { - $this->active = false; + $this->deActivate(); Event::fire(new UserLocked($this->getId())); return $this; } @@ -827,7 +835,7 @@ class User extends BaseEntity */ public function unlock() { - $this->active = true; + $this->activate(); return $this; } @@ -1385,8 +1393,31 @@ SQL; $this->bio = $bio; } + public function activate():void { + if(!$this->active) { + $this->active = true; + $this->spam_type = self::SpamTypeHam; + Event::fire(new UserSpamStateUpdated( + $this->getId() + ) + ); + } + } + + public function deActivate():void { + if( $this->active) { + $this->active = false; + $this->spam_type = self::SpamTypeSpam; + Event::fire(new UserSpamStateUpdated( + $this->getId() + ) + ); + } + } + /** * @return $this + * @throws \Exception */ public function verifyEmail() { @@ -1449,11 +1480,36 @@ SQL; } /** - * @ORM\PostPersist + * @ORM\postPersist */ - public function inserted($args) + public function postPersist($args) { - Event::fire(new UserCreated($this->getId(), $args)); + Event::fire(new UserCreated($this->getId())); + } + + /** + * @ORM\preRemove + */ + public function preRemove($args) + { + } + + /** + * @ORM\preUpdate + * @param PreUpdateEventArgs $args + */ + public function preUpdate(PreUpdateEventArgs $args) + { + if($this->spam_type != self::SpamTypeNone && + !$args->hasChangedField("active") && + ($args->hasChangedField("bio") || $args->hasChangedField("email"))) { + // enqueue user for spam re checker + $this->resetSpamTypeClassification(); + Event::fire(new UserSpamStateUpdated($this->getId())); + } + if($args->hasChangedField("email")) { + // record email change + } } /** @@ -1554,4 +1610,43 @@ SQL; $this->reset_password_requests->clear(); } + /** + * @return string|null + */ + public function getSpamType(): ?string + { + return $this->spam_type; + } + + /** + * @param string $spam_type + * @throws ValidationException + */ + public function setSpamType(string $spam_type): void + { + if(!in_array($spam_type, self::ValidSpamTypes)) + throw new ValidationException(sprintf("Not valid %s spam type value.", $spam_type)); + $this->spam_type = $spam_type; + } + + + public function resetSpamTypeClassification():void{ + $this->spam_type = self::SpamTypeNone; + } + + /** + * @return bool + */ + public function isHam():bool{ + return $this->spam_type == self::SpamTypeHam; + } + + /** + * @return bool + */ + public function isSpam():bool{ + return $this->spam_type == self::SpamTypeSpam; + } + + } \ No newline at end of file diff --git a/app/libs/Auth/Repositories/ISpamEstimatorFeedRepository.php b/app/libs/Auth/Repositories/ISpamEstimatorFeedRepository.php new file mode 100644 index 00000000..35d30826 --- /dev/null +++ b/app/libs/Auth/Repositories/ISpamEstimatorFeedRepository.php @@ -0,0 +1,22 @@ +file_handle = fopen($filename, "r"); + if(!$this->file_handle) + throw new InvalidArgumentException; + } + + /** + * @param string $content + * @return array + */ + public static function load($content) + { + $data = str_getcsv($content,"\n" ); + $lines = array(); + $header = array(); + $idx = 0; + foreach($data as $row) + { + $row = str_getcsv($row, ","); + ++$idx; + if($idx === 1) { $header = $row; continue;} + $line = array(); + for($i=0; $i < count($header); $i++){ + $line[$header[$i]] = trim($row[$i]); + } + $lines[] = $line; + + } //parse the items in rows + return $lines; + } + + function __destruct() { + if($this->file_handle) fclose($this->file_handle); + } + + /** + * @return array|bool + */ + function getLine(){ + if (!feof($this->file_handle) ) { + return fgetcsv($this->file_handle, 1024); + } + return false; + } +} \ No newline at end of file diff --git a/database/migrations/Version20200306133045.php b/database/migrations/Version20200306133045.php new file mode 100644 index 00000000..a4ff90b3 --- /dev/null +++ b/database/migrations/Version20200306133045.php @@ -0,0 +1,91 @@ +hasTable("users") && !$builder->hasColumn("users","spam_type") ) { + $builder->table('users', function (Table $table) { + $table->string('spam_type')->setNotnull(true)->setDefault('None'); + }); + } + + if(!$schema->hasTable("users_spam_estimator_feed")) { + $builder->create('users_spam_estimator_feed', function (Table $table) { + $table->increments('id'); + $table->timestamps(); + $table->string("first_name", 100)->setNotnull(false); + $table->string("last_name", 100)->setNotnull(false); + $table->string("email", 255)->setNotnull(false); + $table->unique("email"); + $table->text("bio")->setNotnull(false); + $table->string('spam_type')->setNotnull(true)->setDefault('None'); + }); + } + + if(!$schema->hasTable("users_deleted")) { + $builder->create('users_deleted', function (Table $table) { + $table->increments('id'); + $table->timestamps(); + $table->string("first_name", 100)->setNotnull(false); + $table->string("last_name", 100)->setNotnull(false); + $table->string("email", 255)->setNotnull(false); + $table->unique("email"); + $table->bigInteger("performer_id")->setUnsigned(true); + $table->index("performer_id", "performer_id"); + $table->foreign("users", "performer_id", "id", ["onDelete" => "CASCADE"]); + }); + } + + if(!$schema->hasTable("users_email_changed")) { + $builder->create('users_email_changed', function (Table $table) { + $table->increments('id'); + $table->timestamps(); + $table->string("former_email", 255)->setNotnull(false); + $table->string("new_email", 255)->setNotnull(false); + $table->bigInteger("user_id")->setUnsigned(true); + $table->index("user_id", "user_id"); + $table->foreign("users", "user_id", "id", ["onDelete" => "CASCADE"]); + $table->bigInteger("performer_id")->setUnsigned(true); + $table->index("performer_id", "performer_id"); + $table->foreign("users", "performer_id", "id", ["onDelete" => "CASCADE"]); + }); + } + } + + /** + * @param Schema $schema + */ + public function down(Schema $schema) + { + $builder = new Builder($schema); + $builder->dropIfExists("users_email_changed"); + $builder->dropIfExists("users_deleted"); + $builder->dropIfExists("users_spam_estimator_feed"); + } +} diff --git a/database/migrations/Version20200306135446.php b/database/migrations/Version20200306135446.php new file mode 100644 index 00000000..3f51e668 --- /dev/null +++ b/database/migrations/Version20200306135446.php @@ -0,0 +1,59 @@ +addSql($sql); + + $sql = <<addSql($sql); + + // reset spam state to Ham + $sql = <<addSql($sql); + } + + /** + * @param Schema $schema + */ + public function down(Schema $schema) + { + + } +} diff --git a/public/assets/js/admin/users.js b/public/assets/js/admin/users.js index e2829418..a36cdede 100644 --- a/public/assets/js/admin/users.js +++ b/public/assets/js/admin/users.js @@ -11,6 +11,7 @@ function UsersCrud(urls, perPage) { '' + '' + '' + + '' + ' ' + actions + '' + ''); @@ -21,6 +22,7 @@ function UsersCrud(urls, perPage) { 'td.user-fname': 'user.first_name', 'td.user-lname': 'user.last_name', 'td.user-email': 'user.email', + 'td.user-spam-type': 'user.spam_type', 'td.user-last-login': function (arg) { if (arg.item.last_login_date == null) return 'N/A'; return moment.unix(arg.item.last_login_date).format(); @@ -80,7 +82,7 @@ UsersCrud.prototype.init = function () { url: url, contentType: "application/json; charset=utf-8", success: function (data, textStatus, jqXHR) { - + _this.loadPage(); }, error: function (jqXHR, textStatus, errorThrown) { ajaxError(jqXHR, textStatus, errorThrown); diff --git a/resources/views/admin/edit-user.blade.php b/resources/views/admin/edit-user.blade.php index e007a9a9..f406886c 100644 --- a/resources/views/admin/edit-user.blade.php +++ b/resources/views/admin/edit-user.blade.php @@ -178,7 +178,10 @@ /> Email Verified? - +
+ + +
diff --git a/resources/views/admin/users.blade.php b/resources/views/admin/users.blade.php index 6e1707bc..e5f81082 100644 --- a/resources/views/admin/users.blade.php +++ b/resources/views/admin/users.blade.php @@ -30,6 +30,7 @@ Email Active Last Login Date + Spam Type   diff --git a/resources/views/emails/user_spammer_process_result.blade.php b/resources/views/emails/user_spammer_process_result.blade.php new file mode 100644 index 00000000..d7636bb2 --- /dev/null +++ b/resources/views/emails/user_spammer_process_result.blade.php @@ -0,0 +1,18 @@ + + + + + + +

+

+

+

Cheers,
{!! Config::get('app.tenant_name') !!} Support Team

+ + \ No newline at end of file