From a6baa51471fd5d21826a750116cc632c2b0bd706 Mon Sep 17 00:00:00 2001 From: Richard Gomez Date: Wed, 26 Jun 2024 21:58:17 -0400 Subject: [PATCH] feat(gui): server-side pagination --- .../src/app/appmain/aadobjects.service.ts | 102 ++++++++++++------ .../src/app/appmain/appmain.module.ts | 5 +- .../appmain/devices/devices.component.html | 2 +- .../app/appmain/devices/devices.component.ts | 31 ++++-- .../app/appmain/groups/groups.component.html | 2 +- .../app/appmain/groups/groups.component.ts | 33 ++++-- .../app/appmain/users/users.component.html | 2 +- .../src/app/appmain/users/users.component.ts | 26 ++++- roadrecon/roadtools/roadrecon/server.py | 58 ++++++++-- 9 files changed, 195 insertions(+), 66 deletions(-) diff --git a/roadrecon/frontend/src/app/appmain/aadobjects.service.ts b/roadrecon/frontend/src/app/appmain/aadobjects.service.ts index 1b15526e..13b0c752 100644 --- a/roadrecon/frontend/src/app/appmain/aadobjects.service.ts +++ b/roadrecon/frontend/src/app/appmain/aadobjects.service.ts @@ -1,11 +1,11 @@ import { Injectable } from '@angular/core'; -import { HttpClient } from '@angular/common/http'; +import {HttpClient, HttpParams} from '@angular/common/http'; import { environment } from '../../environments/environment'; import { Router, Resolve, RouterStateSnapshot, - ActivatedRouteSnapshot -} from '@angular/router'; + ActivatedRouteSnapshot, Params +} from '@angular/router'; import { Observable, of, EMPTY } from 'rxjs'; import { mergeMap, take } from 'rxjs/operators'; import {SortDirection} from '@angular/material/sort'; @@ -245,8 +245,9 @@ export interface AuthorizationPolicy { export interface PaginationParams { page: number; pageSize: number; - sortField: string; - sortDirection: SortDirection; + sortField?: string; + sortDirection?: SortDirection; + contains?: string; } @Injectable({ @@ -260,12 +261,7 @@ export class DatabaseService { public getUsers(pagination?: PaginationParams): Observable { return this.http.get(environment.apibase + 'users', { - params: { - page: pagination.page || this.defaultPage, - per_page: pagination.pageSize || this.defaultPageSize, - sort: pagination.sortField, - direction: pagination.sortDirection - } + params: this.toHttpParams(pagination) }); } @@ -273,32 +269,40 @@ export class DatabaseService { return this.http.get(environment.apibase + 'users/'+ id); } - public getDevices(): Observable { - return this.http.get(environment.apibase + 'devices'); + public getDevices(pagination?: PaginationParams): Observable { + return this.http.get(environment.apibase + 'devices', + { params: this.toHttpParams(pagination) } + ); } public getDevice(id): Observable { return this.http.get(environment.apibase + 'devices/'+ id); } - public getGroups(): Observable { - return this.http.get(environment.apibase + 'groups'); + public getGroups(pagination?: PaginationParams): Observable { + return this.http.get(environment.apibase + 'groups', + { params: this.toHttpParams(pagination) } + ); } public getGroup(id): Observable { return this.http.get(environment.apibase + 'groups/'+ id); } - public getAdministrativeUnits(): Observable { - return this.http.get(environment.apibase + 'administrativeunits'); + public getAdministrativeUnits(pagination?: PaginationParams): Observable { + return this.http.get(environment.apibase + 'administrativeunits', + { params: this.toHttpParams(pagination) } + ); } public getAdministrativeUnit(id): Observable { return this.http.get(environment.apibase + 'administrativeunits/'+ id); } - public getServicePrincipals(): Observable { - return this.http.get(environment.apibase + 'serviceprincipals'); + public getServicePrincipals(pagination?: PaginationParams): Observable { + return this.http.get(environment.apibase + 'serviceprincipals', + { params: this.toHttpParams(pagination) } + ); } public getServicePrincipal(id): Observable { @@ -309,20 +313,26 @@ export class DatabaseService { return this.http.get(environment.apibase + 'serviceprincipals-by-appid/'+ id); } - public getApplications(): Observable { - return this.http.get(environment.apibase + 'applications'); + public getApplications(pagination?: PaginationParams): Observable { + return this.http.get(environment.apibase + 'applications', + { params: this.toHttpParams(pagination) } + ); } public getApplication(id): Observable { return this.http.get(environment.apibase + 'applications/'+ id); } - public getDirectoryRoles(): Observable { - return this.http.get(environment.apibase + 'directoryroles'); + public getDirectoryRoles(pagination?: PaginationParams): Observable { + return this.http.get(environment.apibase + 'directoryroles', + { params: this.toHttpParams(pagination) } + ); } - public getRoleDefinitions(): Observable { - return this.http.get(environment.apibase + 'roledefinitions'); + public getRoleDefinitions(pagination?: PaginationParams): Observable { + return this.http.get(environment.apibase + 'roledefinitions', + { params: this.toHttpParams(pagination) } + ); } public getTenantStats(): Observable { @@ -332,17 +342,21 @@ export class DatabaseService { public getTenantDetail(): Observable { return this.http.get(environment.apibase + 'tenantdetails'); } - + public getDirectorySetting(): Observable { return this.http.get(environment.apibase + 'directorysettings'); } - public getAuthorizationPolicies(): Observable { - return this.http.get(environment.apibase + 'authorizationpolicies'); + public getAuthorizationPolicies(pagination?: PaginationParams): Observable { + return this.http.get(environment.apibase + 'authorizationpolicies', + { params: this.toHttpParams(pagination) } + ); } - public getAppRoles(): Observable { - return this.http.get(environment.apibase + 'approles'); + public getAppRoles(pagination?: PaginationParams): Observable { + return this.http.get(environment.apibase + 'approles', + { params: this.toHttpParams(pagination) } + ); } public getAppRolesByResource(spid): Observable { @@ -353,12 +367,32 @@ export class DatabaseService { return this.http.get(environment.apibase + 'approles_by_principal/' + pid); } - public getMfa(): Observable { - return this.http.get(environment.apibase + 'mfa'); + public getMfa(pagination?: PaginationParams): Observable { + return this.http.get(environment.apibase + 'mfa', + { params: this.toHttpParams(pagination) } + ); + } + + public getOAuth2Permissions(pagination?: PaginationParams): Observable { + return this.http.get(environment.apibase + 'oauth2permissions', + { params: this.toHttpParams(pagination) } + ); } - public getOAuth2Permissions(): Observable { - return this.http.get(environment.apibase + 'oauth2permissions'); + toHttpParams(params?: PaginationParams): HttpParams { + let httpParams = new HttpParams(); + httpParams = httpParams.set('page', params?.page || this.defaultPage); + httpParams = httpParams.set('per_page', params?.pageSize || this.defaultPageSize); + if (params?.sortField) { + httpParams = httpParams.set('sort', params.sortField); + } + if (params?.sortDirection) { + httpParams = httpParams.set('direction', params.sortDirection); + } + if (params?.contains) { + httpParams = httpParams.set('contains', params.contains); + } + return httpParams; } } diff --git a/roadrecon/frontend/src/app/appmain/appmain.module.ts b/roadrecon/frontend/src/app/appmain/appmain.module.ts index bc973c05..8e70a91f 100644 --- a/roadrecon/frontend/src/app/appmain/appmain.module.ts +++ b/roadrecon/frontend/src/app/appmain/appmain.module.ts @@ -37,7 +37,7 @@ import { DevicesdialogComponent } from './devices/devicesdialog/devicesdialog.co import { JsonFormatDirective } from './json-format.directive'; import { ConfigComponent } from './config/config.component'; import { NgxWebstorageModule } from 'ngx-webstorage'; -import { FormsModule } from '@angular/forms'; +import {FormsModule, ReactiveFormsModule} from '@angular/forms'; import { MfaComponent } from './mfa/mfa.component'; import { Oauth2permissionsComponent } from './oauth2permissions/oauth2permissions.component'; @@ -85,7 +85,8 @@ import { Oauth2permissionsComponent } from './oauth2permissions/oauth2permission MatButtonModule, MatTooltipModule, MatSlideToggleModule, - NgxWebstorageModule.forRoot({'prefix':'RT'}), + NgxWebstorageModule.forRoot({'prefix': 'RT'}), + ReactiveFormsModule, ], exports: [ UsersComponent, diff --git a/roadrecon/frontend/src/app/appmain/devices/devices.component.html b/roadrecon/frontend/src/app/appmain/devices/devices.component.html index 713538f0..004a9857 100644 --- a/roadrecon/frontend/src/app/appmain/devices/devices.component.html +++ b/roadrecon/frontend/src/app/appmain/devices/devices.component.html @@ -1,6 +1,6 @@
- + diff --git a/roadrecon/frontend/src/app/appmain/devices/devices.component.ts b/roadrecon/frontend/src/app/appmain/devices/devices.component.ts index b8f0ba1e..4f40e271 100644 --- a/roadrecon/frontend/src/app/appmain/devices/devices.component.ts +++ b/roadrecon/frontend/src/app/appmain/devices/devices.component.ts @@ -2,7 +2,8 @@ import { AfterViewInit, Component, OnInit, ViewChild } from '@angular/core'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; import { MatTable, MatTableDataSource } from '@angular/material/table'; -import { DatabaseService, DevicesItem } from '../aadobjects.service' +import {DatabaseService, DevicesItem, UsersItem} from '../aadobjects.service'; +import {FormControl} from '@angular/forms'; // import @Component({ selector: 'app-devices', @@ -14,6 +15,7 @@ export class DevicesComponent implements AfterViewInit, OnInit { @ViewChild(MatPaginator) paginator: MatPaginator; @ViewChild(MatSort) sort: MatSort; @ViewChild(MatTable) table: MatTable; + readonly filterControl = new FormControl(''); dataSource: MatTableDataSource; constructor(private service: DatabaseService) { } @@ -23,18 +25,35 @@ export class DevicesComponent implements AfterViewInit, OnInit { ngOnInit() { this.dataSource = new MatTableDataSource(); - this.service.getDevices().subscribe((data: DevicesItem[]) => this.dataSource.data = data); + this.loadData(); } ngAfterViewInit() { this.dataSource.sort = this.sort; this.dataSource.paginator = this.paginator; this.table.dataSource = this.dataSource; + + this.filterControl.valueChanges.subscribe(() => this.loadData()); + this.sort.sortChange.subscribe(() => { + this.paginator.pageIndex = 0; // Reset to first page on sort change + this.loadData(); + }); + this.paginator.page.subscribe(() => this.loadData()); } - applyFilter(filterValue: string) { - filterValue = filterValue.trim(); // Remove whitespace - filterValue = filterValue.toLowerCase(); // Datasource defaults to lowercase matches - this.dataSource.filter = filterValue; + loadData() { + let filterValue = this.filterControl.value; + if (filterValue) { + filterValue = filterValue.trim().toLowerCase(); + } + this.service.getDevices( + { + page: this.paginator?.pageIndex, + pageSize: this.paginator?.pageSize, + sortField: this.sort?.active, + sortDirection: this.sort?.direction, + contains: filterValue, + } + ).subscribe((data: DevicesItem[]) => this.dataSource.data = data); } } diff --git a/roadrecon/frontend/src/app/appmain/groups/groups.component.html b/roadrecon/frontend/src/app/appmain/groups/groups.component.html index eb88b2f6..87e3fcf0 100644 --- a/roadrecon/frontend/src/app/appmain/groups/groups.component.html +++ b/roadrecon/frontend/src/app/appmain/groups/groups.component.html @@ -1,6 +1,6 @@
- +
diff --git a/roadrecon/frontend/src/app/appmain/groups/groups.component.ts b/roadrecon/frontend/src/app/appmain/groups/groups.component.ts index 2749edf8..6a926137 100644 --- a/roadrecon/frontend/src/app/appmain/groups/groups.component.ts +++ b/roadrecon/frontend/src/app/appmain/groups/groups.component.ts @@ -3,7 +3,8 @@ import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; import { MatTable, MatTableDataSource } from '@angular/material/table'; // import { GroupsDataSource } from './groups-datasource'; -import { DatabaseService, GroupsItem } from '../aadobjects.service' +import {DatabaseService, DevicesItem, GroupsItem} from '../aadobjects.service'; +import {FormControl} from '@angular/forms'; // import @Component({ selector: 'app-groups', @@ -15,6 +16,7 @@ export class GroupsComponent implements AfterViewInit, OnInit { @ViewChild(MatPaginator) paginator: MatPaginator; @ViewChild(MatSort) sort: MatSort; @ViewChild(MatTable) table: MatTable; + readonly filterControl = new FormControl(''); dataSource: MatTableDataSource; constructor(private service: DatabaseService) { } @@ -24,18 +26,33 @@ export class GroupsComponent implements AfterViewInit, OnInit { ngOnInit() { this.dataSource = new MatTableDataSource(); - this.service.getGroups().subscribe((data: GroupsItem[]) => this.dataSource.data = data); + this.loadData(); } ngAfterViewInit() { - this.dataSource.sort = this.sort; - this.dataSource.paginator = this.paginator; this.table.dataSource = this.dataSource; + + this.filterControl.valueChanges.subscribe(() => this.loadData()); + this.sort.sortChange.subscribe(() => { + this.paginator.pageIndex = 0; // Reset to first page on sort change + this.loadData(); + }); + this.paginator.page.subscribe(() => this.loadData()); } - applyFilter(filterValue: string) { - filterValue = filterValue.trim(); // Remove whitespace - filterValue = filterValue.toLowerCase(); // Datasource defaults to lowercase matches - this.dataSource.filter = filterValue; + loadData() { + let filterValue = this.filterControl.value; + if (filterValue) { + filterValue = filterValue.trim().toLowerCase(); + } + this.service.getGroups( + { + page: this.paginator?.pageIndex, + pageSize: this.paginator?.pageSize, + sortField: this.sort?.active, + sortDirection: this.sort?.direction, + contains: filterValue, + } + ).subscribe((data: GroupsItem[]) => this.dataSource.data = data); } } diff --git a/roadrecon/frontend/src/app/appmain/users/users.component.html b/roadrecon/frontend/src/app/appmain/users/users.component.html index 6a9725e8..905da53a 100644 --- a/roadrecon/frontend/src/app/appmain/users/users.component.html +++ b/roadrecon/frontend/src/app/appmain/users/users.component.html @@ -1,6 +1,6 @@
- +
diff --git a/roadrecon/frontend/src/app/appmain/users/users.component.ts b/roadrecon/frontend/src/app/appmain/users/users.component.ts index e1cd9fcc..fdf39ec4 100644 --- a/roadrecon/frontend/src/app/appmain/users/users.component.ts +++ b/roadrecon/frontend/src/app/appmain/users/users.component.ts @@ -3,8 +3,10 @@ import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; import { MatTable, MatTableDataSource } from '@angular/material/table'; // import { UsersDataSource } from './users-datasource'; -import { DatabaseService, UsersItem } from '../aadobjects.service' +import { DatabaseService, UsersItem } from '../aadobjects.service'; import { LocalStorageService } from 'ngx-webstorage'; +import {MatInput} from '@angular/material/input'; +import {FormControl} from '@angular/forms'; // import @Component({ selector: 'app-users', @@ -16,6 +18,7 @@ export class UsersComponent implements AfterViewInit, OnInit { @ViewChild(MatPaginator) paginator: MatPaginator; @ViewChild(MatSort) sort: MatSort; @ViewChild(MatTable) table: MatTable; + readonly filterControl = new FormControl(''); dataSource: MatTableDataSource; constructor(private service: DatabaseService, private localSt:LocalStorageService) { } @@ -50,6 +53,9 @@ export class UsersComponent implements AfterViewInit, OnInit { this.dataSource.paginator = this.paginator; this.table.dataSource = this.dataSource; + this.filterControl.valueChanges.subscribe(() => { + this.loadData(); + }); this.sort.sortChange.subscribe(() => { this.paginator.pageIndex = 0; // Reset to first page on sort change this.loadData(); @@ -60,10 +66,20 @@ export class UsersComponent implements AfterViewInit, OnInit { }); } - applyFilter(filterValue: string) { - filterValue = filterValue.trim(); // Remove whitespace - filterValue = filterValue.toLowerCase(); // Datasource defaults to lowercase matches - this.dataSource.filter = filterValue; + loadData() { + let filterValue = this.filterControl.value; + if (filterValue) { + filterValue = filterValue.trim().toLowerCase(); + } + this.service.getUsers( + { + page: this.paginator?.pageIndex, + pageSize: this.paginator?.pageSize, + sortField: this.sort?.active, + sortDirection: this.sort?.direction, + contains: filterValue, + } + ).subscribe((data: UsersItem[]) => this.dataSource.data = data); } loadData() { diff --git a/roadrecon/roadtools/roadrecon/server.py b/roadrecon/roadtools/roadrecon/server.py index 351be12b..b625e031 100644 --- a/roadrecon/roadtools/roadrecon/server.py +++ b/roadrecon/roadtools/roadrecon/server.py @@ -1,9 +1,11 @@ -from typing import Optional, TypeVar, Type +import sqlite3 +from typing import Optional, TypeVar, Type, List from flask import Flask, request, jsonify, abort, send_from_directory, redirect, send_file from flask_sqlalchemy import SQLAlchemy from flask_marshmallow import Marshmallow from flask_cors import CORS +from flask_sqlalchemy.pagination import Pagination from marshmallow_sqlalchemy import ModelConverter from marshmallow import fields from roadtools.roadlib.metadef.database import User, JSON, Group, DirectoryRole, ServicePrincipal, AppRoleAssignment, \ @@ -11,9 +13,13 @@ RoleDefinition, ModelBase import os import argparse -from sqlalchemy import func, and_, or_, select, desc, asc +from sqlalchemy import func, and_, or_, select, desc, asc, Column import mimetypes +from sqlalchemy.dialects import sqlite +from sqlalchemy.orm import InstrumentedAttribute, Query +from sqlalchemy.sql.elements import SQLCoreOperations + app = Flask(__name__) app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False @@ -204,8 +210,11 @@ def get_gui(path): @app.route("/api/users", methods=["GET"]) def get_users(): - order_by = get_sort(User, default_field="userPrincipalName") - users = db.paginate(db.session.query(User).order_by(order_by)) + users = paginated_query( + User, + default_sort_field="userPrincipalName", + searchable_columns=[User.displayName, User.mail, User.mobile, User.userPrincipalName] + ) result = users_schema.dump(users) return jsonify(result) @@ -219,8 +228,15 @@ def user_detail(id): @app.route("/api/devices", methods=["GET"]) def get_devices(): - order_by = get_sort(Device, default_field="displayName") - devices = db.paginate(db.session.query(Device).order_by(order_by)) + devices = paginated_query( + Device, + default_sort_field="displayName", + searchable_columns=[ + Device.displayName, Device.deviceId, Device.objectId, + Device.deviceManufacturer, Device.deviceModel, Device.deviceOSType, + Device.deviceOSVersion, Device.deviceTrustType, + ] + ) result = devices_schema.dump(devices) return jsonify(result) @@ -242,7 +258,13 @@ def user_groups(id): @app.route("/api/groups", methods=["GET"]) def get_groups(): - groups = db.paginate(db.session.query(Group)) + groups = paginated_query( + Group, + default_sort_field="displayName", + searchable_columns=[ + Group.displayName, Group.description, Group.groupTypes, Group.mail, + ] + ) result = groups_schema.dump(groups) return jsonify(result) @@ -564,7 +586,27 @@ def get_stats(): return jsonify(stats) M = TypeVar('M', bound=ModelBase) -def get_sort(model: Type[M], default_field: str): + +def paginated_query(model: Type[M], default_sort_field: str, searchable_columns: List[InstrumentedAttribute]) -> Pagination: + query = db.session.query(model) + query = apply_contains(query, *searchable_columns) + order_by = build_order_by(model, default_field=default_sort_field) + query = query.order_by(order_by) + + # print(query.statement.compile(dialect=sqlite.dialect(), compile_kwargs={"literal_binds": True})) + + return db.paginate(query) + +def apply_contains(query: Query, *columns: InstrumentedAttribute) -> Query: + criteria = request.args.get("contains") + if not criteria: + return query + + return query.where(or_( + *[col.contains(criteria) for col in columns] + )) + +def build_order_by(model: Type[M], default_field: str): order_by = default_field sort = request.args.get("sort")