Files
directus/tests-blackbox/routes/items/m2o.seed.ts
ian f1a8e0446f Fix duplicated results and functions in nested filters (#14798)
* Speed query up by reusing existing aliases which reduces table joins

* Use subquery in top level m2o to remove duplicates

* Fix linting

* Apply distinct on primary key field in subqueries

* Use distinct instead as there are only primary keys

* Apply subquery on top level

* Try remove sub sub query

* Test if working for all vendors

* Add support for _none and _some

* Use subquery only when field depth > 1

* Add tests

* Use original table names for columns with functions (#14690)

* Use original table names for columns with functions

* Extract filter function path parsing as shared util

* Fix filter function path when adding node

* Pass the originalCollectionName into filter functions

* Update unit test

* Replace functions within deep GraphQL

* Fix invalid operator error for _none and _some

* Add filter function tests

* Revert triggering for all vendors

* Simplify aliasMap

* Replace functions in filter within GraphQL aggregate query

* Add API support for filtering of alias field

* Mark schema as optional

* Shift logical operators upwards

* Separate recursive parseFilter

* Rework shifting of logical operators

* Error on invalid usage of _none and _some

* Use inner join to preserve sort order

* Run tests for all vendors

* Reuse aliasMap for sort and filter

* Sort on top level query

* Remove unnecessary limit on wrapper query

* Refactor applyQuery options

* Remove duplicates from nested multi relational sort

* Fix offset in MSSQL requiring OrderBy

* Disable schema cache

* Use inner query only for nested sort or multi relational filter

* Fix MSSQL duplicate order column

* Use inner query only for multi relational

* Additional integration tests

* Order within partition for multi relational sorts

* Rename to directus_row_number

* Fix unit test

* Add base sort and filter tests

* Fix Oracle uppercased rowNumber column

* Fix unit test

* Fix top level query sort with function

* Parse functions in inner query

* Increase clarity with knex.ref()

* Remove sort filter for top level primary key

* Fix unit test

* Bypass queries with groupBy

* Add collection to aliasMap to fix functions in nested sort

* Fix multi relational sort with functions

* Add tests for filter and sort with functions

* Fix accidental deletion of brackets

* Fix top level alias filter node interface

* Update M2M sort tests

* Add M2A tests

* Cast m2a primary key as varchar2 for oracle

* Enable filtering tests for M2A

* Fix prototype polluting assignment in aliasMap

* Remove unnecessary currentKey

* Simplify code to increase readability

Co-authored-by: Brainslug <br41nslug@users.noreply.github.com>

* Fix linting and missing 'this' error

* Revert optional chaining

* Add mysql5 to tests

* Fix mysql5 missing rowNumber()

* Overcome indexing delays in MySQL5

* Verify MySQL5 sorting is in order as the result count varies between runs

* Skip joining when sorting field already exists

* Simplify variable assignment

Co-authored-by: Azri Kahar <42867097+azrikahar@users.noreply.github.com>

* Fix linting

* Reduce duplicate logic with vars

* Transform _func fields in GraphQL only for valid functions

* Fix unit test

* Fix unsupported date_part() in CrDB

Co-authored-by: Brainslug <br41nslug@users.noreply.github.com>
Co-authored-by: Roger Stringer <roger@directus.io>
Co-authored-by: Azri Kahar <42867097+azrikahar@users.noreply.github.com>
Co-authored-by: Rijk van Zanten <rijkvanzanten@me.com>
2022-12-21 11:56:18 -05:00

316 lines
8.5 KiB
TypeScript

import vendors from '@common/get-dbs-to-test';
import {
CreateCollection,
CreateField,
CreateFieldM2O,
CreateItem,
DeleteCollection,
SeedFunctions,
PrimaryKeyType,
PRIMARY_KEY_TYPES,
} from '@common/index';
import { CachedTestsSchema, TestsSchema, TestsSchemaVendorValues } from '@query/filter';
import { set } from 'lodash';
import { seedAllFieldTypesStructure, seedAllFieldTypesValues, getTestsAllTypesSchema } from './seed-all-field-types';
import { seedRelationalFields } from './seed-relational-fields';
export const collectionCountries = 'test_items_m2o_countries';
export const collectionStates = 'test_items_m2o_states';
export const collectionCities = 'test_items_m2o_cities';
export type Country = {
id?: number | string;
name: string;
test_datetime?: string;
};
export type State = {
id?: number | string;
name: string;
country_id?: number | string | null;
test_datetime?: string;
};
export type City = {
id?: number | string;
name: string;
state_id?: number | string | null;
};
export function getTestsSchema(pkType: PrimaryKeyType, seed?: string): TestsSchema {
const schema: TestsSchema = {
[`${collectionCountries}_${pkType}`]: {
id: {
field: 'id',
type: pkType,
isPrimaryKey: true,
filters: true,
possibleValues: SeedFunctions.generatePrimaryKeys(pkType, {
quantity: 2,
seed: `collectionCountries${seed}`,
incremental: true,
}),
},
name: {
field: 'name',
type: 'string',
filters: true,
possibleValues: ['United States', 'Malaysia'],
},
...getTestsAllTypesSchema(),
},
};
schema[`${collectionStates}_${pkType}`] = {
id: {
field: 'id',
type: pkType,
isPrimaryKey: true,
filters: false,
possibleValues: SeedFunctions.generatePrimaryKeys(pkType, {
quantity: 4,
seed: `collectionStates${seed}`,
incremental: true,
}),
},
name: {
field: 'name',
type: 'string',
filters: false,
possibleValues: ['Washington', 'California', 'Johor', 'Sarawak'],
},
country_id: {
field: 'country_id',
type: pkType,
filters: false,
possibleValues: SeedFunctions.generatePrimaryKeys(pkType, {
quantity: 2,
seed: `collectionCountries${seed}`,
incremental: true,
}),
children: schema[`${collectionCountries}_${pkType}`],
relatedCollection: `${collectionCountries}_${pkType}`,
},
};
schema[`${collectionCities}_${pkType}`] = {
id: {
field: 'id',
type: pkType,
isPrimaryKey: true,
filters: false,
possibleValues: SeedFunctions.generatePrimaryKeys(pkType, {
quantity: 8,
seed: `collectionCities${seed}`,
incremental: true,
}),
},
name: {
field: 'name',
type: 'string',
filters: false,
possibleValues: [
'Seattle',
'Spokane',
'Los Angeles',
'San Francisco',
'Johor Bahru',
'Muar',
'Kota Kinabalu',
'Sandakan',
],
},
state_id: {
field: 'state_id',
type: pkType,
filters: false,
possibleValues: SeedFunctions.generatePrimaryKeys(pkType, {
quantity: 4,
seed: `collectionCountries${seed}`,
incremental: true,
}),
children: schema[`${collectionStates}_${pkType}`],
relatedCollection: `${collectionStates}_${pkType}`,
},
};
return schema;
}
export const seedDBStructure = () => {
it.each(vendors)(
'%s',
async (vendor) => {
for (const pkType of PRIMARY_KEY_TYPES) {
try {
const localCollectionCountries = `${collectionCountries}_${pkType}`;
const localCollectionStates = `${collectionStates}_${pkType}`;
const localCollectionCities = `${collectionCities}_${pkType}`;
// Delete existing collections
await DeleteCollection(vendor, { collection: localCollectionCities });
await DeleteCollection(vendor, { collection: localCollectionStates });
await DeleteCollection(vendor, { collection: localCollectionCountries });
// Create countries collection
await CreateCollection(vendor, {
collection: localCollectionCountries,
primaryKeyType: pkType,
});
await CreateField(vendor, {
collection: localCollectionCountries,
field: 'name',
type: 'string',
});
// Create states collection
await CreateCollection(vendor, {
collection: localCollectionStates,
primaryKeyType: pkType,
});
await CreateField(vendor, {
collection: localCollectionStates,
field: 'name',
type: 'string',
});
await CreateFieldM2O(vendor, {
collection: localCollectionStates,
field: 'country_id',
primaryKeyType: pkType,
otherCollection: localCollectionCountries,
});
// Create cities collection
await CreateCollection(vendor, {
collection: localCollectionCities,
primaryKeyType: pkType,
});
await CreateField(vendor, {
collection: localCollectionCities,
field: 'name',
type: 'string',
});
await CreateFieldM2O(vendor, {
collection: localCollectionCities,
field: 'state_id',
primaryKeyType: pkType,
otherCollection: localCollectionStates,
});
await seedAllFieldTypesStructure(vendor, localCollectionCountries);
await seedAllFieldTypesStructure(vendor, localCollectionStates);
await seedAllFieldTypesStructure(vendor, localCollectionCities);
expect(true).toBeTruthy();
} catch (error) {
expect(error).toBeFalsy();
}
}
},
600000
);
};
export const seedDBValues = async (cachedSchema: CachedTestsSchema, vendorSchemaValues: TestsSchemaVendorValues) => {
await Promise.all(
vendors.map(async (vendor) => {
for (const pkType of PRIMARY_KEY_TYPES) {
const schema = cachedSchema[pkType];
const localCollectionCountries = `${collectionCountries}_${pkType}`;
const localCollectionStates = `${collectionStates}_${pkType}`;
const localCollectionCities = `${collectionCities}_${pkType}`;
// Create countries
const itemCountries = [];
for (let i = 0; i < schema[localCollectionCountries].id.possibleValues.length; i++) {
const country: Country = {
name: schema[localCollectionCountries].name.possibleValues[i],
};
if (pkType === 'string') {
country.id = schema[localCollectionCountries].id.possibleValues[i];
}
itemCountries.push(country);
}
const countries = await CreateItem(vendor, {
collection: localCollectionCountries,
item: itemCountries,
});
const countriesIDs = countries.map((country: Country) => country.id);
set(vendorSchemaValues, `${vendor}.${localCollectionCountries}.id`, countriesIDs);
// Create states
const itemStates = [];
for (let i = 0; i < schema[localCollectionStates].id.possibleValues.length; i++) {
const state: State = {
name: schema[localCollectionStates].name.possibleValues[i],
country_id: countriesIDs[i % countriesIDs.length],
};
if (pkType === 'string') {
state.id = schema[localCollectionStates].id.possibleValues[i];
}
itemStates.push(state);
}
const states = await CreateItem(vendor, {
collection: localCollectionStates,
item: itemStates,
});
const statesIDs = states.map((state: State) => state.id);
set(vendorSchemaValues, `${vendor}.${localCollectionStates}.id`, statesIDs);
set(vendorSchemaValues, `${vendor}.${localCollectionStates}.country_id.id`, countriesIDs);
// Create cities
const itemCities = [];
for (let i = 0; i < schema[localCollectionCities].id.possibleValues.length; i++) {
const city: City = {
name: schema[localCollectionCities].name.possibleValues[i],
state_id: statesIDs[i % statesIDs.length],
};
if (pkType === 'string') {
city.id = schema[localCollectionCities].id.possibleValues[i];
}
itemCities.push(city);
}
const cities = await CreateItem(vendor, {
collection: localCollectionCities,
item: itemCities,
});
const citiesIDs = cities.map((city: City) => city.id);
set(vendorSchemaValues, `${vendor}.${localCollectionCities}.id`, citiesIDs);
set(vendorSchemaValues, `${vendor}.${localCollectionCities}.state_id.id`, statesIDs);
set(vendorSchemaValues, `${vendor}.${localCollectionCities}.state_id.country_id.id`, countriesIDs);
await seedAllFieldTypesValues(vendor, localCollectionCountries, pkType);
await seedAllFieldTypesValues(vendor, localCollectionStates, pkType);
await seedAllFieldTypesValues(vendor, localCollectionCities, pkType);
await seedRelationalFields(vendor, localCollectionStates, pkType, schema[localCollectionStates]);
await seedRelationalFields(vendor, localCollectionCities, pkType, schema[localCollectionCities]);
}
})
);
};