Code Coverage
 
Classes and Traits
Functions and Methods
Lines
Total
0.00% covered (danger)
0.00%
0 / 1
22.22% covered (danger)
22.22%
4 / 18
CRAP
40.40% covered (warning)
40.40%
80 / 198
SessionController
0.00% covered (danger)
0.00%
0 / 1
22.22% covered (danger)
22.22%
4 / 18
493.88
40.40% covered (warning)
40.40%
80 / 198
 __construct
100.00% covered (success)
100.00%
1 / 1
1
100.00% covered (success)
100.00%
4 / 4
 index
100.00% covered (success)
100.00%
1 / 1
1
100.00% covered (success)
100.00%
1 / 1
 superAdminIndex
100.00% covered (success)
100.00%
1 / 1
1
100.00% covered (success)
100.00%
1 / 1
 userIndex
0.00% covered (danger)
0.00%
0 / 1
2
0.00% covered (danger)
0.00%
0 / 1
 externalIndex
0.00% covered (danger)
0.00%
0 / 1
13.12
31.25% covered (danger)
31.25%
5 / 16
 store
0.00% covered (danger)
0.00%
0 / 1
2.01
85.71% covered (success)
85.71%
18 / 21
 update
0.00% covered (danger)
0.00%
0 / 1
2.01
86.96% covered (success)
86.96%
20 / 23
 fileAvailable
0.00% covered (danger)
0.00%
0 / 1
2.06
75.00% covered (success)
75.00%
9 / 12
 upload
0.00% covered (danger)
0.00%
0 / 1
2.05
76.92% covered (success)
76.92%
10 / 13
 dedup
0.00% covered (danger)
0.00%
0 / 1
6
0.00% covered (danger)
0.00%
0 / 8
 download
0.00% covered (danger)
0.00%
0 / 1
90
0.00% covered (danger)
0.00%
0 / 19
 handleBlackList
100.00% covered (success)
100.00%
1 / 1
3
100.00% covered (success)
100.00%
12 / 12
 createNewOperation
0.00% covered (danger)
0.00%
0 / 1
6
0.00% covered (danger)
0.00%
0 / 5
 getDefaultStructure
0.00% covered (danger)
0.00%
0 / 1
2
0.00% covered (danger)
0.00%
0 / 1
 getNameOperation
0.00% covered (danger)
0.00%
0 / 1
2
0.00% covered (danger)
0.00%
0 / 2
 sendEmail
0.00% covered (danger)
0.00%
0 / 1
12
0.00% covered (danger)
0.00%
0 / 20
 copyOperation
0.00% covered (danger)
0.00%
0 / 1
42
0.00% covered (danger)
0.00%
0 / 34
 getEmailsExternalUsers
0.00% covered (danger)
0.00%
0 / 1
6
0.00% covered (danger)
0.00%
0 / 5
1<?php
2
3namespace Qmp\Laravel\Deduplication\Controllers;
4
5use Carbon\Carbon;
6use Illuminate\Http\JsonResponse;
7use Illuminate\Http\Response;
8use Illuminate\Http\Request;
9use Illuminate\Support\Arr;
10use Illuminate\Support\Collection;
11use Illuminate\Support\Facades\File;
12use Illuminate\Support\Facades\Log;
13use Illuminate\Support\Facades\Redis;
14use Illuminate\Support\Facades\Storage;
15use Illuminate\Support\Str;
16use MongoDB\Driver\WriteConcern;
17use Qmp\Laravel\Deduplication\Mail\DedupMailableFactory;
18use Qmp\Laravel\Deduplication\Models\Session;
19use Qmp\Laravel\Deduplication\Models\SessionData;
20use Qmp\Laravel\Deduplication\Objects\DedupRunner;
21use Qmp\Laravel\Deduplication\Objects\HandleFile;
22use Qmp\Laravel\DeduplicationQueue\Jobs\Deduplication;
23use Qmp\Laravel\Mail\MailSender;
24use Qmp\Laravel\MicroService\Controllers\AbstractMicroServiceController;
25use Qmp\Laravel\MicroService\Client\Tools\Request as ClientRequest;
26use Qmp\Laravel\MicroService\Client\Client;
27use Qmp\Laravel\MongoLow\MongoPipelineDate;
28
29class SessionController extends AbstractMicroServiceController
30{
31    //subFolders
32    const UPLOAD = 'uploaded';
33    const DOWNLOAD = 'download';
34    const BLACKLIST = 'blacklist';
35
36    const CUSTOMERHASHEDLIST = 'customerHashedList';
37
38    const USER_ID_PLACEHOLDER = "%%USERID%%";
39
40    protected $operationTypeId = 3;
41
42    protected $disk;
43    protected $handleFile;
44
45    protected $validations = [
46        'name' => 'required',
47        'start' => 'required',
48        'deadline' => 'required',
49        'sites' => 'required|json',
50        'hashType' => 'nullable|in:md5,sha1,sha256',
51        'availableOn' => 'required',
52        'description' => 'present',
53        'advertiser' => 'present',
54        'customerHashedList' => 'nullable|mimes:txt,csv,zip'
55    ];
56
57    /**
58     * Undocumented function
59     *
60     * @param Request $request
61     * @param HandleFile $handleFile
62     */
63    public function __construct(Request $request, HandleFile $handleFile)
64    {
65        parent::__construct($request);
66
67        $this->handleFile = $handleFile;
68        $this->disk = Storage::disk('deduplication');
69    }
70
71    /**
72     * Display a listing of the resource.
73     *
74     * @return \Illuminate\Http\JsonResponse
75     */
76    public function index()
77    {
78        return response()->json($this->assignedIndex());
79    }
80
81    /**
82     * Undocumented function
83     *
84     * @return void
85     */
86    protected function superAdminIndex()
87    {
88        return Session::all();
89    }
90
91    /**
92     * Undocumented function
93     *
94     * @param [type] $ids
95     * @return void
96     */
97    protected function userIndex($ids)
98    {
99
100        return Session::whereIn('id', $ids)->get();
101    }
102
103    /**
104     * Undocumented function
105     *
106     * @return void
107     */
108    protected function externalIndex()
109    {
110        $httprequest = ClientRequest::createObject('service_entities', 'entity/from-url', ['body' => [
111            'url' => $this->httpOrigin
112        ]]);
113        $entity = Client::systemSend('post', $httprequest)->content;
114
115        if (is_array($entity) && isset($entity['id'])) {
116            $httprequest = ClientRequest::createObject('service_campaigns', 'campaign/get-operations-by-entity/' . $this->operationTypeId . '/' . $entity['id']);
117            $response = Client::systemSend('get', $httprequest)->content;
118
119            if (isset($response['datas']) && !empty($response['datas'])) {
120                $operations = $response['datas'];
121
122                $query = '[
123                    {
124                        "$match": {
125                            "$and": [
126                                {"id": {"$in": ' . json_encode($operations) . '}},
127                                {"sites.user_id":{"$eq":' . self::USER_ID_PLACEHOLDER . '}},
128                                {
129                                    "$or" : [
130                                        {"availableUntil":null},
131                                        {"availableUntil": { "$gt": ' . MongoPipelineDate::toJson() . ' }}
132                                    ]
133                                }
134                            ]
135                        }
136                    },
137                    {
138                      "$addFields":{
139                        "sites":{
140                          "$filter":{"input":"$sites","as":"site","cond":{"$eq":[' . self::USER_ID_PLACEHOLDER . ',"$$site.user_id"]}}
141                        }
142                      }
143                    },
144                    {
145                      "$unset":["blackLists","error","lastDedup","_id","report","updated_at","user_id"]
146                    }
147                  ]';
148
149                $externalAggregation = str_replace(self::USER_ID_PLACEHOLDER, (int)$this->userId, $query);
150                $pipeline = Session::pipeline();
151
152                return $pipeline->raw($externalAggregation)->all();
153            }
154        }
155
156        return [];
157    }
158
159    /**
160     * Store a newly created resource in storage.
161     *
162     * @param \Illuminate\Http\Request $request
163     * @return \Illuminate\Http\JsonResponse
164     */
165    public function store(Request $request)
166    {
167        $request->validate($this->validations);
168
169        try {
170
171            $sites = json_decode($request->sites, true);
172
173            $id = uniqid();
174            $session = Session::create([
175                'id' => $id,
176                'name' => $request->name,
177                'start' => $request->start,
178                'deadLine' => $request->deadline,
179                'sites' => $sites,
180                'hashType' => $request->hashType,
181                'blackLists' => [],
182                'fileAvailable' => false,
183                'report' => null,
184                'lastDedup' => null,
185                'user_id' => $this->userId,
186                'availableOn' => $request->availableOn,
187                'description' => $request->description,
188                'advertiser' => $request->advertiser,
189                'availableUntil' => $request->availableUntil
190            ]);
191
192            $this->handleBlackList($session, $request);
193
194            Redis::publish('deduplication', json_encode(['action' => 'save-session', 'sessionId' => $session->id, 'message' => $session->name . " - save-session"]));
195            return response()->json(['status' => 'ok', 'datas' => $session->ToArray()], Response::HTTP_CREATED);
196        } catch (\Exception $e) {
197            Log::debug('Unable to store entity :' . var_export(['message' => $e->getMessage(), 'line' => $e->getLine(), 'file' => $e->getFile()], true));
198            return response()->json(['status' => 'ko', 'message' => $e->getMessage()], Response::HTTP_UNPROCESSABLE_ENTITY);
199        }
200    }
201
202    /**
203     * Update the specified resource in storage.
204     *
205     * @param \Illuminate\Http\Request $request
206     * @return \Illuminate\Http\JsonResponse
207     */
208    public function update(Request $request, $id)
209    {
210        $request->validate(array_merge($this->validations, [
211            'removeOldAdvertiserFile' => 'required|in:true,false',
212            'hashType' => 'required|in:sha1,md5,sha256',
213            'description' => 'required',
214            'advertiser' => 'required',
215        ]));
216
217        try {
218            $this->checkOperationAssignment($request->id);
219            $session = Session::findOrFail($id);
220
221            $sites = json_decode($request->sites, true);
222            
223            $ids = array_diff(array_column($session->sites, 'id'), array_column($sites, 'id'));
224
225            SessionData::whereIn('site_id', $ids)->where('session_id', $session->id)->delete(["writeConcern" => new writeConcern(0)]);
226
227            $session->update([
228                'name' => $request->name,
229                'start' => $request->start,
230                'deadLine' => $request->deadline,
231                'hashType' => $request->hashType,
232                'sites' => $sites,
233                'availableOn' => $request->availableOn,
234                'description' => $request->description,
235                'advertiser' => $request->advertiser,
236                'availableUntil' => $request->availableUntil
237            ]);
238
239            $this->handleBlackList($session, $request);
240
241            Redis::publish('deduplication', json_encode(['action' => 'update-session', 'sessionId' => $session->id, 'message' => $session->name . " - update-session"]));
242            return response()->json(['status' => 'ok', 'datas' => $session->refresh()->toArray()], Response::HTTP_OK);
243        } catch (\Exception $e) {
244            Log::debug('Unable to update entity :' . var_export(['message' => $e->getMessage(), 'line' => $e->getLine(), 'file' => $e->getFile()], true));
245            return response()->json(['status' => 'ko', 'errors' => $e->getMessage()], $this->getHttpCode($e));
246        }
247    }
248
249    /**
250     * Undocumented function
251     *
252     * @param Request $request
253     * @param [type] $id
254     * @return \Illuminate\Http\JsonResponse
255     */
256    public function fileAvailable(Request $request, $id): JsonResponse
257    {
258        $request->validate([
259            'id' => 'required|in:' . $id,
260            // 'fileAvailable' => 'required|boolean'
261        ]);
262
263        try {
264
265            $session = Session::findOrFail($id);
266            $session->creatingFiles = true;
267            $session->save();
268
269            $dedupRunner = new DedupRunner($session->id);
270            Deduplication::dispatch($dedupRunner->onlyFiles());
271
272            Redis::publish('deduplication', json_encode(['action' => 'file-available', 'sessionId' => $session->id, 'message' => $session->name . " - file-available"]));
273            return response()->json(['status' => 'ok']);
274        } catch (\Exception $e) {
275            Log::debug('Unable to update entity :' . var_export(['message' => $e->getMessage(), 'line' => $e->getLine(), 'file' => $e->getFile()], true));
276            return response()->json(['status' => 'ko', 'errors' => $e->getMessage()], Response::HTTP_UNPROCESSABLE_ENTITY);
277        }
278    }
279
280
281    /**
282     * Upload site list file
283     *
284     * @param Request $request
285     * @param $session_id
286     * @return \Illuminate\Http\JsonResponse
287     */
288    public function upload(Request $request, $session_id)
289    {
290        $request->validate([
291            'site_id' => 'required',
292            'dedup_file' => 'required|mimes:txt,csv,zip'
293        ]);
294
295        try {
296
297            $session = Session::findOrFail($session_id);
298
299            $name = $request->file('dedup_file')->getClientOriginalName();
300
301            $input_file = $this->handleFile
302                ->config($request->file('dedup_file'), $session->hashType)
303                ->store($session_id . '/' . self::UPLOAD . '/' . $request->site_id, $name);
304
305            $session->setKeyFilter('sites.$id', 'input_file', $input_file, [(int)$request->site_id]);
306
307            Redis::publish('deduplication', json_encode(['action' => 'upload-file', 'sessionId' => $session->id, 'siteId' => $request->site_id, 'message' => $session->name . " - upload-file " . $request->site_id]));
308            return response()->json(['status' => 'ok', 'file' => $input_file], Response::HTTP_OK);
309        } catch (\Exception $e) {
310            Log::debug('Unable to upload entity :' . var_export(['message' => $e->getMessage(), 'line' => $e->getLine(), 'file' => $e->getFile()], true));
311            return response()->json(['status' => 'ko', 'errors' => $e->getMessage()], Response::HTTP_UNPROCESSABLE_ENTITY);
312        }
313    }
314
315    /**
316     * Start dedup of session
317     *
318     * @param $session_id
319     * @return \Illuminate\Http\JsonResponse
320     */
321    public function dedup($session_id)
322    {
323        try {
324            $session = Session::findOrFail($session_id);
325
326            $dedupRunner = new DedupRunner($session->id);
327            Deduplication::dispatch($dedupRunner);
328
329            Redis::publish('deduplication', json_encode(['action' => 'start-dedup', 'sessionId' => $session->id, 'message' => $session->name . " - start-dedup"]));
330            return response()->json(['status' => 'ok'], Response::HTTP_OK);
331        } catch (\Exception $e) {
332            Log::debug('Unable to launch dedup :' . var_export(['message' => $e->getMessage(), 'line' => $e->getLine(), 'file' => $e->getFile()], true));
333            return response()->json(['status' => 'ko', 'errors' => $e->getMessage()], Response::HTTP_UNPROCESSABLE_ENTITY);
334        }
335    }
336
337    /**
338     * Download output site file
339     *
340     * @param Request $request
341     * @param $session_id
342     * @return \Illuminate\Http\JsonResponse
343     */
344    public function download(Request $request, $session_id)
345    {
346        $request->validate([
347            'id' => 'required',
348            'type' => 'required|in:input,output',
349            'format' => 'nullable|in:zip'
350        ]);
351
352        try {
353            $session = Session::findOrFail($session_id);
354
355            $recursive = collect($session)->recursive();
356            $query = $recursive->get('sites')->where('id', $request->id);
357
358            if (strtolower($this->authType) === 'external') {
359                $query = $query->where('user_id', $this->userId);
360            }
361
362            $site = $query->first();
363
364            if (!$site) {
365                throw new \Exception('Unauthorized !');
366            }
367
368            $type = $request->type === 'input' ? 'input' : 'output';
369            $format = $request->format === 'zip' ? 'zip' : 'file';
370            $path = $type === 'input' ? $site->dotGet($type . '_file.base') : $site->dotGet($type . '_file.' . $format);
371
372            if (strtolower($this->authType) === 'external' && $request->type == 'output') {
373                $session->setKeyFilter('sites.$id.output_file', 'downloaded', true, [(int)$request->id]);
374            }
375
376            return response()->json(['status' => 'ok', 'path' => $path], Response::HTTP_OK);
377        } catch (\Exception $e) {
378            Log::debug('Unable to launch dedup :' . var_export(['message' => $e->getMessage(), 'line' => $e->getLine(), 'file' => $e->getFile()], true));
379            return response()->json(['status' => 'ko', 'errors' => $e->getMessage()], Response::HTTP_UNPROCESSABLE_ENTITY);
380        }
381    }
382
383    /**
384     * Handle blacklist Files
385     * @param Session $session
386     * @param Request $request
387     * @throws \Exception
388     */
389    private function handleBlackList(Session $session, Request $request)
390    {
391        $file = [];
392        if ($request->removeOldAdvertiserFile === 'true') {
393            $this->disk->deleteDirectory($session->id . '/' . self::BLACKLIST . '/' . self::CUSTOMERHASHEDLIST);
394            $session->pull('blackLists', ['type' => 'customerHashedList']);
395        }
396
397        if ($request->hasFile('customerHashedList')) {
398            $file['type'] = 'customerHashedList';
399            $name = $request->file('customerHashedList')->getClientOriginalName();
400            $file['input_file'] = $this->handleFile
401                ->config($request->file('customerHashedList'), $request->hashType)
402                ->store($session->id . '/' . self::BLACKLIST . '/' . self::CUSTOMERHASHEDLIST, $name);
403            $session->push('blackLists', $file);
404        }
405    }
406
407    /**
408     * Undocumented function
409     *
410     * @param Request $request
411     * @return void
412     */
413    public function createNewOperation(Request $request)
414    {
415        $request->merge($this->getDefaultStructure());
416
417        $response = $this->store($request);
418
419        if (Str::startsWith($response->status(), '2')) {
420            return $this->GetResponseForCreatedOperation($response->getOriginalContent()['datas']['id']);
421        }
422
423        return $response;
424    }
425
426    /**
427     * Undocumented function
428     *
429     * @return void
430     */
431    private function getDefaultStructure()
432    {
433        return include(__DIR__ . '/../DefaultStructure/SessionStructure.php');
434    }
435
436    public function getNameOperation($id)
437    {
438        $session = Session::findOrFail($id, ['id', 'name']);
439
440        return response()->json($session);
441    }
442
443    /**
444     * Send email to publishers
445     *
446     * @param Request $request
447     * @param $session_id
448     * @return \Illuminate\Http\JsonResponse
449     */
450    public function sendEmail(Request $request, $session_id): JsonResponse
451    {
452        $request->validate([
453            'emailType' => 'required|in:inform,relaunch,available'
454        ]);
455
456        $session = Session::findOrFail($session_id);
457
458        try {
459            $emailType = $request->emailType;
460
461            $user = $this->getCurrentUser();
462
463            Carbon::setLocale('fr');
464
465            $data = [
466                'campaign' => $session->name,
467                'date' => Carbon::parse($session->deadLine)->translatedFormat('l d F Y H:i'),
468                'dateAvailable' => Carbon::parse($session->availableOn)->translatedFormat('l d F Y'),
469                'hash' => $session->hashType,
470                'user' => $user['firstname'] . ' ' . $user['lastname']
471            ];
472
473            $mailable = DedupMailableFactory::$emailType($data);
474
475            $ids = $emailType === 'available' ? $session->getUsersIdsWithFiles() : $session->getUsersIdsNoFiles();
476
477            $this->getEmailsExternalUsers($ids)->map(function ($email) use ($mailable) {
478                MailSender::model($mailable)->to($email)->dispatch();
479            });
480
481            return response()->json(['status' => 'ok']);
482        } catch (\Exception $e) {
483            Log::debug('Unable send email to publishers :' . var_export(['message' => $e->getMessage(), 'line' => $e->getLine(), 'file' => $e->getFile()], true));
484            return response()->json(['status' => 'ko', 'errors' => $e->getMessage()], Response::HTTP_UNPROCESSABLE_ENTITY);
485        }
486    }
487
488    /**
489     * @param \Illuminate\Http\Request $request
490     * @return \Illuminate\Http\JsonResponse
491     */
492    public function copyOperation(Request $request): JsonResponse
493    {
494        $request->validate([
495            'id' => 'required',
496            'campaign_id' => 'required'
497        ]);
498
499        $session = Session::findOrFail($request->id);
500
501        try {
502            $now = (new \DateTime())->format('Y-m-d');
503
504            $newSession = $session->toArray();
505
506            $newSession = array_merge($newSession, [
507                'id' => uniqid(),
508                'name' => $newSession['name'] . ' copy',
509                'start' => $now,
510                'deadLine' => $now,
511                'availableOn' => $now,
512                'availableUntil' => $now,
513                'fileAvailable' => false,
514                'report' => null,
515                'lastDedup' => null,
516                'user_id' => (int) $this->userId,
517                'total_steps' => null,
518                'step' => null
519            ]);
520
521            if (Arr::has($newSession, 'blackLists')) {
522                $newSession['blackLists'] = array_map(function($list) use ($session, $newSession) {
523                    $list['input_file']['base'] = str_replace($session->id, $newSession['id'], $list['input_file']['base']);
524                    $list['input_file']['file'] = str_replace($session->id, $newSession['id'], $list['input_file']['file']);
525                    return $list;
526                }, $newSession['blackLists']);
527            }
528
529            $newSession['sites'] = array_map(function ($site) use ($session, $newSession) {
530                if(Arr::has($site, 'input_file')) {
531                    $site['input_file']['base'] = str_replace($session->id, $newSession['id'], $site['input_file']['base']);
532                    $site['input_file']['file'] = str_replace($session->id, $newSession['id'], $site['input_file']['file']);
533                }
534                return Arr::except($site, ['output_file']);
535            }, $newSession['sites']);
536
537            $newSession = Session::create($newSession);
538
539            if (file_exists($this->disk->path($session->id.'/'.self::UPLOAD))) {
540                File::copyDirectory($this->disk->path($session->id.'/'.self::UPLOAD), $this->disk->path($newSession->id.'/'.self::UPLOAD));
541            }
542
543            if (file_exists($this->disk->path($session->id.'/'.self::BLACKLIST))) {
544                File::copyDirectory($this->disk->path($session->id.'/'.self::BLACKLIST), $this->disk->path($newSession->id.'/'.self::BLACKLIST));
545            }
546
547            return response()->json(['status' => 'ok', 'operation_id' => $newSession->id]);
548
549        } catch (\Exception $e) {
550            Log::debug('Unable to copy session :' . var_export(['message' => $e->getMessage(), 'line' => $e->getLine(), 'file' => $e->getFile()], true));
551            return response()->json(['status' => 'ko', 'errors' => $e->getMessage()], Response::HTTP_UNPROCESSABLE_ENTITY);
552        }
553    }
554
555    /**
556     * @param array $ids
557     * @return \Illuminate\Support\Collection
558     * @throws \Exception
559     */
560    protected function getEmailsExternalUsers(array $ids): Collection
561    {
562        $request = ClientRequest::createObject('service_external_users', 'users');
563        $response = Client::systemSend('get', $request);
564
565        if (!Str::startsWith($response->code, 2)) {
566            throw new \Exception('unable to get externals users list ');
567        }
568
569        return collect($response->content)->whereIn('id', $ids)->pluck('email');
570    }
571
572}