Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

254 validation endpoint #523

Merged
merged 8 commits into from
Oct 25, 2022
Merged

254 validation endpoint #523

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e0bd433
add validation endpoint
hornmo Sep 21, 2022
ce3d5dd
more tests
hornmo Sep 23, 2022
fd21177
Merge branch 'g3_master' into 254-validation-endpoint
hornmo Oct 10, 2022
1af5541
adjust error objects
hornmo Oct 11, 2022
91d4c95
use service in upload validation
hornmo Oct 12, 2022
0dafea3
service tests
hornmo Oct 18, 2022
72f186f
fix tests & adjust rest validation
hornmo Oct 24, 2022
cadd7bc
clean up
hornmo Oct 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions server/gokbg3/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,10 @@ dependencies {
compile group: 'org.aspectj', name: 'aspectjweaver', version: '1.8.13'
compile group: 'joda-time', name: 'joda-time', version: '2.9.9'
compile group: 'org.apache.commons', name: 'commons-compress', version: '1.16.1'
compile group: 'commons-validator', name: 'commons-validator', version: '1.7'
compile group: 'com.opencsv', name: 'opencsv', version: '5.6'
compile group: 'org.codehaus.groovy.modules.http-builder', name: 'http-builder', version: '0.7.1'
compile group: 'com.github.ladutsko', name: 'isbn-core', version: '1.0.17'
compile 'org.opensearch:opensearch:1.3.3'
compile 'org.opensearch.client:opensearch-rest-high-level-client:1.3.3'
compile 'org.locationtech.spatial4j:spatial4j:0.6'
Expand Down
123 changes: 61 additions & 62 deletions server/gokbg3/grails-app/conf/application.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ grails.plugin.springsecurity.filterChain.chainMap = [
[pattern: '/fwk/**', filters: 'JOINED_FILTERS,-exceptionTranslationFilter'],
[pattern: '/api/**', filters: 'JOINED_FILTERS,-exceptionTranslationFilter'],
[pattern: '/integration/**', filters: 'JOINED_FILTERS,-exceptionTranslationFilter'],
[pattern: '/validation/**', filters: 'JOINED_FILTERS,-exceptionTranslationFilter,-basicAuthenticationFilter,-basicExceptionTranslationFilter,-securityContextPersistenceFilter,-rememberMeAuthenticationFilter'],
[pattern: '/admin/bulkLoadUsers', filters: 'JOINED_FILTERS,-exceptionTranslationFilter'],
[pattern: '/rest/login', filters: 'JOINED_FILTERS,-exceptionTranslationFilter,-basicAuthenticationFilter,-basicExceptionTranslationFilter,-authenticationProcessingFilter,-securityContextPersistenceFilter,-rememberMeAuthenticationFilter'],
[pattern: '/rest/**/**', filters: 'JOINED_FILTERS,-exceptionTranslationFilter,-basicAuthenticationFilter,-basicExceptionTranslationFilter,-securityContextPersistenceFilter,-rememberMeAuthenticationFilter'],
Expand All @@ -44,69 +45,67 @@ grails.plugin.springsecurity.filterChain.chainMap = [
]

grails.plugin.springsecurity.controllerAnnotations.staticRules = [
[pattern: '/admin/**', access: ['ROLE_ADMIN', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/file/**', access: ['ROLE_ADMIN', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/info', access: ['ROLE_ADMIN', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/monitoring/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/login/auth', access: ['permitAll']],
[pattern: '/greenmail/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/', access: ['permitAll']],
[pattern: '/index', access: ['permitAll']],
[pattern: '/notFound', access: ['permitAll']],
[pattern: '/index.gsp', access: ['permitAll']],
[pattern: '/register/**', access: ['permitAll']],
[pattern: '/public/**', access: ['permitAll']],
[pattern: '/package/**', access: ['permitAll']],
[pattern: '/packages/**', access: ['permitAll']],
[pattern: '/admin/**', access: ['ROLE_ADMIN', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/file/**', access: ['ROLE_ADMIN', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/info', access: ['ROLE_ADMIN', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/monitoring/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/login/auth', access: ['permitAll']],
[pattern: '/greenmail/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/', access: ['permitAll']],
[pattern: '/index', access: ['permitAll']],
[pattern: '/notFound', access: ['permitAll']],
[pattern: '/index.gsp', access: ['permitAll']],
[pattern: '/register/**', access: ['permitAll']],
[pattern: '/public/**', access: ['permitAll']],
[pattern: '/package/**', access: ['permitAll']],
[pattern: '/packages/**', access: ['permitAll']],
[pattern: '/component/identifierConflicts', access: ['ROLE_EDITOR', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/public', access: ['permitAll']],
[pattern: '/error', access: ['permitAll']],
[pattern: '/error/**', access: ['permitAll']],
[pattern: '/globalSearch/**', access: ['ROLE_USER']],
[pattern: '/home/**', access: ['ROLE_USER']],
[pattern: '/assets/**', access: ['permitAll']],
[pattern: '/**/js/**', access: ['permitAll']],
[pattern: '/**/css/**', access: ['permitAll']],
[pattern: '/**/images/**', access: ['permitAll']],
[pattern: '/**/favicon.ico', access: ['permitAll']],
[pattern: '/api/find', access: ['permitAll']],
[pattern: '/api/scroll', access: ['permitAll']],
[pattern: '/api/suggest', access: ['permitAll']],
[pattern: '/api/esconfig', access: ['permitAll']],
[pattern: '/api/capabilities', access: ['permitAll']],
[pattern: '/api/downloadUpdate', access: ['permitAll']],
[pattern: '/api/checkUpdate', access: ['permitAll']],
[pattern: '/api/isUp', access: ['permitAll']],
[pattern: '/api/userData', access: ['permitAll']],
[pattern: '/api/refdata', access: ['ROLE_USER']],
[pattern: '/api/show', access: ['ROLE_USER']],
[pattern: '/api/namespaces', access: ['permitAll']],
[pattern: '/api/groups', access: ['permitAll']],
[pattern: '/api/elasticsearchTunnel', access: ['permitAll']],
[pattern: '/api/retrieveZdbCandidates', access: ['permitAll']],
[pattern: '/integration/**', access: ['permitAll']],
[pattern: '/fwk/**', access: ['ROLE_USER']],
[pattern: '/user/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/user/search', access: ['ROLE_ADMIN', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/user/edit/**', access: ['ROLE_ADMIN', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/role/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/securityInfo/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/registrationCode/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/aclClass/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/aclSid/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/aclObjectIdentity/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/aclEntry/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/oai', access: ['permitAll']],
[pattern: '/oai/**', access: ['permitAll']],
[pattern: '/rest/login', access: ['permitAll']],
// [pattern: '/rest/roles', access: ['permitAll']],
//[pattern: '/rest/curatoryGroups', access: ['permitAll']],
//[pattern: '/rest/curatoryGroups/**', access: ['permitAll']],
[pattern: '/rest/refdata', access: ['permitAll']],
[pattern: '/rest/refdata/**', access: ['permitAll']],
[pattern: '/rest/**', access: ['ROLE_USER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/oauth/**', access: ['permitAll']],
[pattern: '/coreference/**', access: ['permitAll']]
[pattern: '/public', access: ['permitAll']],
[pattern: '/error', access: ['permitAll']],
[pattern: '/error/**', access: ['permitAll']],
[pattern: '/globalSearch/**', access: ['ROLE_USER']],
[pattern: '/home/**', access: ['ROLE_USER']],
[pattern: '/assets/**', access: ['permitAll']],
[pattern: '/**/js/**', access: ['permitAll']],
[pattern: '/**/css/**', access: ['permitAll']],
[pattern: '/**/images/**', access: ['permitAll']],
[pattern: '/**/favicon.ico', access: ['permitAll']],
[pattern: '/api/find', access: ['permitAll']],
[pattern: '/api/scroll', access: ['permitAll']],
[pattern: '/api/suggest', access: ['permitAll']],
[pattern: '/api/esconfig', access: ['permitAll']],
[pattern: '/api/capabilities', access: ['permitAll']],
[pattern: '/api/downloadUpdate', access: ['permitAll']],
[pattern: '/api/checkUpdate', access: ['permitAll']],
[pattern: '/api/isUp', access: ['permitAll']],
[pattern: '/api/userData', access: ['permitAll']],
[pattern: '/api/refdata', access: ['ROLE_USER']],
[pattern: '/api/show', access: ['ROLE_USER']],
[pattern: '/api/namespaces', access: ['permitAll']],
[pattern: '/api/groups', access: ['permitAll']],
[pattern: '/api/elasticsearchTunnel', access: ['permitAll']],
[pattern: '/api/retrieveZdbCandidates', access: ['permitAll']],
[pattern: '/integration/**', access: ['permitAll']],
[pattern: '/fwk/**', access: ['ROLE_USER']],
[pattern: '/user/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/user/search', access: ['ROLE_ADMIN', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/user/edit/**', access: ['ROLE_ADMIN', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/role/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/securityInfo/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/registrationCode/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/aclClass/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/aclSid/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/aclObjectIdentity/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/aclEntry/**', access: ['ROLE_SUPERUSER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/oai', access: ['permitAll']],
[pattern: '/oai/**', access: ['permitAll']],
[pattern: '/rest/login', access: ['permitAll']],
[pattern: '/rest/refdata', access: ['permitAll']],
[pattern: '/rest/refdata/**', access: ['permitAll']],
[pattern: '/rest/**', access: ['ROLE_USER', 'IS_AUTHENTICATED_FULLY']],
[pattern: '/oauth/**', access: ['permitAll']],
[pattern: '/coreference/**', access: ['permitAll']],
[pattern: '/validation/**', access: ['permitAll']]
]


Expand Down
2 changes: 2 additions & 0 deletions server/gokbg3/grails-app/conf/application.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ grails:
enabled: true
mappings:
/rest/**: inherit
/validation/**: inherit
/api/**: inherit

profile: web
codegen:
Expand Down
1 change: 1 addition & 0 deletions server/gokbg3/grails-app/controllers/gokbg3/UrlMappings.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ class UrlMappings {
"/oai/$id?"(controller: 'oai', action: 'index')
"/resource/show/$type/$id"(controller: 'resource', action: 'show')
"/package"(controller: 'packages')
"/validate"(controller: 'validation')

group "/rest", {
post "/packages/$id/retire"(controller: 'package', namespace: 'rest', action:'retire')
Expand Down
105 changes: 105 additions & 0 deletions server/gokbg3/grails-app/controllers/org/gokb/ValidationController.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
package org.gokb

import grails.converters.*

import org.gokb.cred.IdentifierNamespace

class ValidationController {

def validationService
def TSVIngestionService

def index() {
def result = [
message: "Please select a specific endpoint!",
endpoints: [
kbart: [method: 'POST', description: 'Validates full KBART files and returns a report.', contentType: "multipart/form-data", content: 'submissionFile', pars: ['namespace']],
componentName: [method: 'GET', description: 'Validates new component names for form and uniqueness', pars: ['value', 'componentType']],
identifier: [method: 'GET', description: 'Validates an identifier value for a specified namespace', pars: ['value', 'namespace']],
url: [method: 'POST', description: 'Validates a URL', contentType: "application/json", content: [value: '<url>']]
]
]

render result as JSON
}

def kbart() {
def result = [result: 'OK', errors: [:]]
def multipart_file = request.getFile("submissionFile")

if (multipart_file && !multipart_file.isEmpty()) {
def title_id_namespace = null
def file_info = TSVIngestionService.analyseFile(multipart_file.getInputStream())

if (!['UTF-8', 'US-ASCII'].contains(file_info.encoding)) {
result.errors.encoding = [message: "The encoding of this file was identified as ${file_info.encoding}, but must be UTF-8!", messageCode: "kbart.errors.encoding", args: []]
}

if (params.namespace) {
title_id_namespace = params.int('namespace') ? IdentifierNamespace.get(params.int('namespace')) : IdentifierNamespace.findByValue(params.namespace)

if (!title_id_namespace) {
result.errors.namespace = [message: "Unable to reference provided namespace for column title_id!", messageCode: "kbart.errors.namespaceNotFound", args: []]
}
}

result.report = validationService.generateKbartReport(multipart_file.getInputStream(), title_id_namespace)

if (result.report.valid == false) {
result.result = 'ERROR'
}
}
else {
result.result = 'ERROR'
result.errors.file = [message: "No/empty file was supplied as 'submissionFile'!", messageCode: "validation.noFile", args: []]
}

render result as JSON
}

def componentName() {
def result = validationService.checkNewComponentName(params.value, params.componentType)

render result as JSON
}

def identifier() {
def result = [result: 'OK']
def namespace = params.int('namespace') ? IdentifierNamespace.get(params.int('namespace')) : IdentifierNamespace.findByValue(params.namespace)

if (!namespace) {
result.result = 'ERROR'
result.errors = [namespace: [message: "Unable to reference namespace ${params.namespace}", messageCode: "validation.unknownNamespace", pars: [params.namespace]]]
}
else {
def validation_result = validationService.checkIdForNamespace(params.value, namespace)

if (!validation_result) {
result.result = 'ERROR'
result.errors = [value: [message: "Value ${params.value} is not valid", messageCode: "validation.invalid", pars: [params.value]]]
}
}

render result as JSON
}

def url() {
def result = [result: 'OK']
def reqBody = request.JSON

if (reqBody && reqBody.value) {
def validation_result = validationService.checkUrl(reqBody.value)

if (!validation_result) {
result.result = 'ERROR'
result.errors = [value: [message: "Provided value ${reqBody.value} is not a valid URL", messageCode: "validation.urlForm", pars: [reqBody.value]]]
}
}
else {
result.result = 'ERROR'
result.result = [value: [message: "No value provided via JSON object!"]]
}

render result as JSON
}
}
5 changes: 4 additions & 1 deletion server/gokbg3/grails-app/controllers/org/gokb/rest/IdentifierController.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ class IdentifierController {
Identifier obj = null

try {
obj = componentLookupService.lookupOrCreateCanonicalIdentifier(ns.value, reqBody.value,false)
obj = componentLookupService.lookupOrCreateCanonicalIdentifier(ns.value, reqBody.value, false)
}
catch (grails.validation.ValidationException ve) {
log.debug("Identifier ${reqBody} has failed validation!")
Expand All @@ -130,6 +130,9 @@ class IdentifierController {

if (!obj) {
log.debug("Could not create identifier!")
result.message = "Unable to create identifier ${reqBody}"
errors = [value: [[message: messageService.resolveCode('identifier.validation.generic', null, request.locale), baddata: reqBody.value, messageCode: 'identifier.validation.generic']]]
response.status = 400
}
else if ( obj.hasErrors() ) {
errors = messageService.processValidationErrors(obj.errors, request.locale)
Expand Down
1 change: 1 addition & 0 deletions server/gokbg3/grails-app/i18n/messages.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,6 +402,7 @@ identifier.value.noNamespace=No Namespace configured!
identifier.create.error=Could not create Identifier!
identifier.link.unique=This identifier is already connected to this component!
identifier.validate.error=Validation for identifier {0}:{1} failed!
identifier.validation.generic=This indentifier value is invalid!

package.name.notNull=Please provide a name for the Package
package.name.notUnique=A Package with this name already exists!
Expand Down
1 change: 1 addition & 0 deletions server/gokbg3/grails-app/i18n/messages_de.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,7 @@ identifier.value.noNamespace=Kein Namensraum konfiguriert!
identifier.create.error=Konnte keinen Identifikator erstellen!
identifier.link.unique=Dieser Identifikator ist bereits mit dieser Komponente verbunden!
identifier.validate.error=Validierung für diesen Identifikator {0}:{1} fehlgeschlagen!
identifier.validation.generic=Dieser Wert ist ungültig!

package.name.notNull=Bitte geben Sie einen Namen für diese Paket an.
package.name.notUnique=Ein Paket mit diesem Namen existiert bereits!
Expand Down
2 changes: 1 addition & 1 deletion server/gokbg3/grails-app/init/gokbg3/BootStrap.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ class BootStrap {
[value: 'eissn', name: 'e-ISSN', family: 'isxn', pattern: "^\\d{4}\\-\\d{3}[\\dX]\$"],
[value: 'issnl', name: 'ISSN-L', family: 'isxn', pattern: "^\\d{4}\\-\\d{3}[\\dX]\$"],
[value: 'doi', name: 'DOI'],
[value: 'zdb', name: 'ZDB-ID', pattern: "^\\d+-[\\dxX]\$"],
[value: 'zdb', name: 'ZDB-ID', pattern: "^\\d{7,10}-[\\dxX]\$"],
[value: 'isil', name: 'ISIL', pattern: "^(?=[0-9A-Z-]{4,16}\$)[A-Z]{1,4}-[A-Z0-9]{1,11}(-[A-Z0-9]+)?\$"]
]

Expand Down
10 changes: 10 additions & 0 deletions server/gokbg3/grails-app/services/gokbg3/RestMappingService.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -541,9 +541,19 @@ class RestMappingService {
try {
if (ns) {
id = componentLookupService.lookupOrCreateCanonicalIdentifier(ns, i.value)

if (!id) {
result.errors << [message: 'This identifier value is invalid!', baddata: i.value, messageCode: 'identifier.validation.generic']
valid = false
}
}
else {
log.warn("Unable to determine namespace ${ns_val}!")

if (!id) {
result.errors << [message: "Unable to reference namespace ${ns_val}!", baddata: i.value, messageCode: 'identifier.validation.namespace']
valid = false
}
}
}
catch (grails.validation.ValidationException ve) {
Expand Down
Loading