diff --git a/packages/core/database/src/database.ts b/packages/core/database/src/database.ts index 0ba129e3d..18e62090a 100644 --- a/packages/core/database/src/database.ts +++ b/packages/core/database/src/database.ts @@ -216,7 +216,7 @@ export class Database extends EventEmitter implements AsyncEmitter { }); // register database field types - for (const [name, field] of Object.entries(FieldTypes)) { + for (const [name, field] of Object.entries(FieldTypes)) { if (['Field', 'RelationField'].includes(name)) { continue; } diff --git a/packages/core/database/src/fields/array-field.ts b/packages/core/database/src/fields/array-field.ts index 7475a8916..9e7ff6b7f 100644 --- a/packages/core/database/src/fields/array-field.ts +++ b/packages/core/database/src/fields/array-field.ts @@ -1,5 +1,5 @@ -import { BaseColumnFieldOptions, Field } from './field'; import { DataTypes } from 'sequelize'; +import { BaseColumnFieldOptions, Field, ValueParser } from './field'; export class ArrayField extends Field { get dataType() { @@ -28,6 +28,51 @@ export class ArrayField extends Field { super.unbind(); this.off('beforeSave', this.sortValue); } + + buildValueParser(ctx: any) { + return new ArrayValueParser(this, ctx); + } +} + +export class ArrayValueParser extends ValueParser { + async setValue(value: any) { + const { map, set } = this.getOptions(); + const values = this.toArr(value); + if (set.size > 0) { + const filtered = values.map((v) => (map.has(v) ? map.get(v) : v)).filter((v) => set.has(v)); + if (values.length === filtered.length) { + this.value = filtered; + } else { + this.errors.push('No matching option found'); + } + } else { + this.value = values; + } + } + + getOptions() { + const options = this.field.options?.['uiSchema']?.enum || []; + const map = new Map(); + const set = new Set(); + for (const option of options) { + set.add(option.value); + set.add(option.label); + map.set(option.label, option.value); + } + return { map, set }; + } + + toArr(value) { + let values = []; + if (!value) { + values = []; + } else if (typeof value === 'string') { + values = value.split(','); + } else if (Array.isArray(value)) { + values = value; + } + return values; + } } export interface ArrayFieldOptions extends BaseColumnFieldOptions { diff --git a/packages/core/database/src/fields/belongs-to-field.ts b/packages/core/database/src/fields/belongs-to-field.ts index f4d1a8ea5..5d0c71afc 100644 --- a/packages/core/database/src/fields/belongs-to-field.ts +++ b/packages/core/database/src/fields/belongs-to-field.ts @@ -1,7 +1,8 @@ -import lodash, { omit } from 'lodash'; +import { omit } from 'lodash'; import { BelongsToOptions as SequelizeBelongsToOptions, Utils } from 'sequelize'; import { Reference } from '../features/ReferencesMap'; import { checkIdentifier } from '../utils'; +import { ToOneValueParser } from './has-one-field'; import { BaseRelationFieldOptions, RelationField } from './relation-field'; export class BelongsToField extends RelationField { @@ -115,6 +116,10 @@ export class BelongsToField extends RelationField { collection.model.refreshAttributes(); // this.collection.removeIndex([this.options.foreignKey]); } + + buildValueParser(ctx: any) { + return new ToOneValueParser(this, ctx); + } } export interface BelongsToFieldOptions extends BaseRelationFieldOptions, SequelizeBelongsToOptions { diff --git a/packages/core/database/src/fields/belongs-to-many-field.ts b/packages/core/database/src/fields/belongs-to-many-field.ts index b0fc12ac2..7f670e8a3 100644 --- a/packages/core/database/src/fields/belongs-to-many-field.ts +++ b/packages/core/database/src/fields/belongs-to-many-field.ts @@ -2,8 +2,10 @@ import { omit } from 'lodash'; import { BelongsToManyOptions as SequelizeBelongsToManyOptions, Utils } from 'sequelize'; import { Collection } from '../collection'; import { Reference } from '../features/ReferencesMap'; +import { Repository } from '../repository'; import { checkIdentifier } from '../utils'; import { BelongsToField } from './belongs-to-field'; +import { ValueParser } from './field'; import { MultipleRelationFieldOptions, RelationField } from './relation-field'; export class BelongsToManyField extends RelationField { @@ -123,6 +125,48 @@ export class BelongsToManyField extends RelationField { this.clearAccessors(); delete collection.model.associations[this.name]; } + + buildValueParser(ctx: any) { + return new ToManyValueParser(this, ctx); + } +} + +export class ToManyValueParser extends ValueParser { + async setValue(value: any) { + const fieldNames = this.getFileNames(); + console.log('fieldNames', fieldNames, this.ctx.column); + if (this.isInterface('chinaRegion')) { + const repository = this.field.database.getRepository(this.field.target) as Repository; + this.value = await Promise.all( + value.split('/').map(async (v) => { + const instance = await repository.findOne({ filter: { [fieldNames.label]: v.trim() } }); + return instance ? instance.get(fieldNames.value) : v; + }), + ); + } else { + const dataIndex = this.ctx?.column?.dataIndex || []; + if (Array.isArray(dataIndex) && dataIndex.length < 2) { + return; + } + const field = this.ctx.column.dataIndex[1]; + const repository = this.field.database.getRepository(this.field.target) as Repository; + this.value = await Promise.all( + value.split(',').map(async (v) => { + const instance = await repository.findOne({ filter: { [field]: v.trim() } }); + return instance ? instance.get(fieldNames.value) : v; + }), + ); + } + } + + getFileNames() { + const fieldNames = this.field.options?.uiSchema?.['x-component-props']?.['fieldNames'] || {}; + return { label: 'id', value: 'id', ...fieldNames }; + } + + isInterface(name) { + return this.field.options.interface === name; + } } export interface BelongsToManyFieldOptions diff --git a/packages/core/database/src/fields/boolean-field.ts b/packages/core/database/src/fields/boolean-field.ts index ee97a43b6..946c39ef5 100644 --- a/packages/core/database/src/fields/boolean-field.ts +++ b/packages/core/database/src/fields/boolean-field.ts @@ -1,10 +1,42 @@ import { DataTypes } from 'sequelize'; -import { BaseColumnFieldOptions, Field } from './field'; +import { BaseColumnFieldOptions, Field, ValueParser } from './field'; export class BooleanField extends Field { get dataType() { return DataTypes.BOOLEAN; } + + buildValueParser(ctx: any) { + return new BooleanValueParser(this, ctx); + } +} + +export class BooleanValueParser extends ValueParser { + async setValue(value: any) { + // Boolean + if (typeof value === 'boolean') { + this.value = value; + } + // Number + else if (typeof value === 'number' && [0, 1].includes(value)) { + this.value = value === 1; + } + // String + else if (typeof value === 'string') { + if (!value) { + this.value = null; + } + if (['1', 'y', 'yes', 'true', '是'].includes(value.toLowerCase())) { + this.value = true; + } else if (['0', 'n', 'no', 'false', '否'].includes(value.toLowerCase())) { + this.value = false; + } else { + this.errors.push(`${JSON.stringify(this.value)} value invalid`); + } + } else { + this.errors.push(`${JSON.stringify(this.value)} value invalid`); + } + } } export interface BooleanFieldOptions extends BaseColumnFieldOptions { diff --git a/packages/core/database/src/fields/date-field.ts b/packages/core/database/src/fields/date-field.ts index 1a07d04bc..448886365 100644 --- a/packages/core/database/src/fields/date-field.ts +++ b/packages/core/database/src/fields/date-field.ts @@ -1,10 +1,38 @@ +import { moment2str } from '@nocobase/utils'; +import moment, { isDate, isMoment } from 'moment'; import { DataTypes } from 'sequelize'; -import { BaseColumnFieldOptions, Field } from './field'; +import { BaseColumnFieldOptions, Field, ValueParser } from './field'; export class DateField extends Field { get dataType() { return DataTypes.DATE(3); } + + buildValueParser(ctx: any) { + return new DateValueParser(this, ctx); + } +} + +export class DateValueParser extends ValueParser { + async setValue(value: any) { + if (isMoment(value)) { + this.value = value; + } else if (isDate(value)) { + this.value = value; + } else if (typeof value === 'string') { + const props = this.getProps(); + const m = moment(value); + if (m.isValid()) { + this.value = moment2str(m, props); + } else { + this.errors.push('Invalid date'); + } + } + } + + getProps() { + return this.field.options?.uiSchema?.['x-component-props'] || {}; + } } export interface DateFieldOptions extends BaseColumnFieldOptions { diff --git a/packages/core/database/src/fields/field.ts b/packages/core/database/src/fields/field.ts index 65fed3d0e..e8c0d1a13 100644 --- a/packages/core/database/src/fields/field.ts +++ b/packages/core/database/src/fields/field.ts @@ -40,9 +40,9 @@ export abstract class Field { [key: string]: any; constructor(options?: any, context?: FieldContext) { - this.context = context; - this.database = context.database; - this.collection = context.collection; + this.context = context as any; + this.database = this.context.database; + this.collection = this.context.collection; this.options = options || {}; this.init(); } @@ -138,7 +138,7 @@ export abstract class Field { // 排序字段通过 sortable 控制 const sortable = this.collection.options.sortable; if (sortable) { - let sortField: string; + let sortField: any; if (sortable === true) { sortField = 'sort'; } else if (typeof sortable === 'string') { @@ -238,4 +238,33 @@ export abstract class Field { typeToString() { return this.dataType.toString(); } + + buildValueParser(ctx: any) { + return new ValueParser(this, ctx); + } +} + +export class ValueParser { + ctx: any; + field: any; + value: any; + errors = []; + + constructor(field: any, ctx: any) { + this.field = field; + this.ctx = ctx; + } + + toString() { + return this.value; + } + + getValue() { + return this.value; + } + + async setValue(value: any) { + console.log(this.field.name, value); + this.value = value; + } } diff --git a/packages/core/database/src/fields/has-many-field.ts b/packages/core/database/src/fields/has-many-field.ts index 2b00815d8..99183f1b1 100644 --- a/packages/core/database/src/fields/has-many-field.ts +++ b/packages/core/database/src/fields/has-many-field.ts @@ -5,11 +5,12 @@ import { ForeignKeyOptions, HasManyOptions, HasManyOptions as SequelizeHasManyOptions, - Utils, + Utils } from 'sequelize'; import { Collection } from '../collection'; import { Reference } from '../features/ReferencesMap'; import { checkIdentifier } from '../utils'; +import { ToManyValueParser } from './belongs-to-many-field'; import { MultipleRelationFieldOptions, RelationField } from './relation-field'; export interface HasManyFieldOptions extends HasManyOptions { @@ -185,6 +186,10 @@ export class HasManyField extends RelationField { // @ts-ignore collection.model.refreshAttributes(); } + + buildValueParser(ctx: any) { + return new ToManyValueParser(this, ctx); + } } export interface HasManyFieldOptions extends MultipleRelationFieldOptions, SequelizeHasManyOptions { diff --git a/packages/core/database/src/fields/has-one-field.ts b/packages/core/database/src/fields/has-one-field.ts index 799660398..486c14bd0 100644 --- a/packages/core/database/src/fields/has-one-field.ts +++ b/packages/core/database/src/fields/has-one-field.ts @@ -1,16 +1,18 @@ -import lodash, { omit } from 'lodash'; +import { omit } from 'lodash'; import { AssociationScope, DataType, ForeignKeyOptions, HasOneOptions, HasOneOptions as SequelizeHasOneOptions, - Utils, + Utils } from 'sequelize'; import { Collection } from '../collection'; -import { checkIdentifier, snakeCase } from '../utils'; -import { BaseRelationFieldOptions, RelationField } from './relation-field'; import { Reference } from '../features/ReferencesMap'; +import { Repository } from '../repository'; +import { checkIdentifier } from '../utils'; +import { ValueParser } from './field'; +import { BaseRelationFieldOptions, RelationField } from './relation-field'; export interface HasOneFieldOptions extends HasOneOptions { /** @@ -191,6 +193,36 @@ export class HasOneField extends RelationField { // @ts-ignore collection.model.refreshAttributes(); } + + buildValueParser(ctx: any) { + return new ToOneValueParser(this, ctx); + } +} + +export class ToOneValueParser extends ValueParser { + async setValue(value: any) { + const fieldNames = this.getFileNames(); + console.log('fieldNames', fieldNames, this.ctx.column); + const dataIndex = this.ctx?.column?.dataIndex || []; + if (Array.isArray(dataIndex) && dataIndex.length < 2) { + return; + } + const field = this.ctx.column.dataIndex[1]; + const repository = this.field.database.getRepository(this.field.target) as Repository; + const instance = await repository.findOne({ filter: { [field]: value.trim() } }); + if (instance) { + this.value = instance.get(fieldNames.value); + } + } + + getFileNames() { + const fieldNames = this.field.options?.uiSchema?.['x-component-props']?.['fieldNames'] || {}; + return { label: 'id', value: 'id', ...fieldNames }; + } + + isInterface(name) { + return this.field.options.interface === name; + } } export interface HasOneFieldOptions extends BaseRelationFieldOptions, SequelizeHasOneOptions { diff --git a/packages/core/database/src/fields/number-field.ts b/packages/core/database/src/fields/number-field.ts index 4b316b481..6e80ebe8f 100644 --- a/packages/core/database/src/fields/number-field.ts +++ b/packages/core/database/src/fields/number-field.ts @@ -1,7 +1,44 @@ import { DataTypes } from 'sequelize'; -import { BaseColumnFieldOptions, Field } from './field'; +import { BaseColumnFieldOptions, Field, ValueParser } from './field'; -export class IntegerField extends Field { +abstract class NumberField extends Field { + buildValueParser(ctx: any) { + return new NumberValueParser(this, ctx); + } +} + +function percent2float(value: string) { + const index = value.indexOf('.'); + if (index === -1) { + return parseFloat(value) / 100; + } + const repeat = value.length - index - 2; + const v = parseInt('1' + '0'.repeat(repeat)); + return (parseFloat(value) * v) / (100 * v); +} + +export class NumberValueParser extends ValueParser { + async setValue(value: any) { + if (value === null || value === undefined || typeof value === 'number') { + this.value = value; + } + if (typeof value === 'string') { + if (!value) { + this.value = null; + } else if (['n/a', '-'].includes(value.toLowerCase())) { + this.value = null; + } else if (value.endsWith('%')) { + this.value = percent2float(value); + console.log(value, this.value); + } else { + const val = +value; + this.value = isNaN(val) ? null : val; + } + } + } +} + +export class IntegerField extends NumberField { get dataType() { return DataTypes.INTEGER; } @@ -11,7 +48,7 @@ export interface IntegerFieldOptions extends BaseColumnFieldOptions { type: 'integer'; } -export class BigIntField extends Field { +export class BigIntField extends NumberField { get dataType() { return DataTypes.BIGINT; } @@ -21,7 +58,7 @@ export interface BigIntFieldOptions extends BaseColumnFieldOptions { type: 'bigInt'; } -export class FloatField extends Field { +export class FloatField extends NumberField { get dataType() { return DataTypes.FLOAT; } @@ -31,7 +68,7 @@ export interface FloatFieldOptions extends BaseColumnFieldOptions { type: 'float'; } -export class DoubleField extends Field { +export class DoubleField extends NumberField { get dataType() { return DataTypes.DOUBLE; } @@ -41,7 +78,7 @@ export interface DoubleFieldOptions extends BaseColumnFieldOptions { type: 'double'; } -export class RealField extends Field { +export class RealField extends NumberField { get dataType() { return DataTypes.REAL; } @@ -51,7 +88,7 @@ export interface RealFieldOptions extends BaseColumnFieldOptions { type: 'real'; } -export class DecimalField extends Field { +export class DecimalField extends NumberField { get dataType() { return DataTypes.DECIMAL; } diff --git a/packages/core/database/src/fields/string-field.ts b/packages/core/database/src/fields/string-field.ts index db7e60617..92ad3c609 100644 --- a/packages/core/database/src/fields/string-field.ts +++ b/packages/core/database/src/fields/string-field.ts @@ -1,10 +1,44 @@ import { DataTypes } from 'sequelize'; -import { BaseColumnFieldOptions, Field } from './field'; +import { BaseColumnFieldOptions, Field, ValueParser } from './field'; export class StringField extends Field { get dataType() { return DataTypes.STRING; } + + buildValueParser(ctx: any) { + return new StringValueParser(this, ctx); + } +} + +export class StringValueParser extends ValueParser { + async setValue(value: any) { + const { map, set } = this.getOptions(); + if (set.size > 0) { + if (map.has(value)) { + value = map.get(value); + } + if (set.has(value)) { + this.value = value; + } else { + this.errors.push('No matching option found'); + } + } else { + this.value = value; + } + } + + getOptions() { + const options = this.field.options?.['uiSchema']?.enum || []; + const map = new Map(); + const set = new Set(); + for (const option of options) { + set.add(option.value); + set.add(option.label); + map.set(option.label, option.value); + } + return { map, set }; + } } export interface StringFieldOptions extends BaseColumnFieldOptions { diff --git a/packages/core/utils/src/date.ts b/packages/core/utils/src/date.ts index b8eb4b718..a6747cf3c 100644 --- a/packages/core/utils/src/date.ts +++ b/packages/core/utils/src/date.ts @@ -76,3 +76,65 @@ export const str2moment = (value?: string | string[], options: Str2momentOptions ? toMoment(value, options) : value; }; + +const toStringByPicker = (value, picker) => { + if (picker === 'year') { + return value.format('YYYY') + '-01-01T00:00:00.000Z'; + } + if (picker === 'month') { + return value.format('YYYY-MM') + '-01T00:00:00.000Z'; + } + if (picker === 'quarter') { + return value.format('YYYY-MM') + '-01T00:00:00.000Z'; + } + if (picker === 'week') { + return value.format('YYYY-MM-DD') + 'T00:00:00.000Z'; + } + return value.format('YYYY-MM-DD') + 'T00:00:00.000Z'; +}; + +const toGmtByPicker = (value: moment.Moment | moment.Moment[], picker?: any) => { + if (!value) { + return value; + } + if (Array.isArray(value)) { + return value.map((val) => toStringByPicker(val, picker)); + } + if (moment.isMoment(value)) { + return toStringByPicker(value, picker); + } +}; + +export interface Moment2strOptions { + showTime?: boolean; + gmt?: boolean; + picker?: 'year' | 'month' | 'week' | 'quarter'; +} + +export const moment2str = (value?: moment.Moment | moment.Moment[], options: Moment2strOptions = {}) => { + const { showTime, gmt, picker } = options; + if (!value) { + return value; + } + if (showTime) { + return gmt ? toGmt(value) : toLocal(value); + } + return toGmtByPicker(value, picker); +}; + +export const mapDateFormat = function () { + return (props: any) => { + const format = getDefaultFormat(props) as any; + const onChange = props.onChange; + return { + ...props, + format: format, + value: str2moment(props.value, props), + onChange: (value: moment.Moment | moment.Moment[]) => { + if (onChange) { + onChange(moment2str(value, props)); + } + }, + }; + }; +}; diff --git a/packages/plugins/import/package.json b/packages/plugins/import/package.json index 942bcc740..d0b4b47c4 100644 --- a/packages/plugins/import/package.json +++ b/packages/plugins/import/package.json @@ -5,6 +5,9 @@ "license": "AGPL-3.0", "main": "./lib/index.js", "types": "./lib/index.d.ts", + "dependencies": { + "xlsx": "^0.18.5" + }, "devDependencies": { "@nocobase/client": "0.9.0-alpha.2", "@nocobase/test": "0.9.0-alpha.2" diff --git a/packages/plugins/import/src/client/useImportAction.ts b/packages/plugins/import/src/client/useImportAction.ts index 95894073b..f8fa29081 100644 --- a/packages/plugins/import/src/client/useImportAction.ts +++ b/packages/plugins/import/src/client/useImportAction.ts @@ -1,14 +1,12 @@ import { Schema, useFieldSchema, useForm } from '@formily/react'; -import { isEmpty } from '@formily/shared'; import { useActionContext, useAPIClient, useBlockRequestContext, useCollection, useCollectionManager, - useCompile, + useCompile } from '@nocobase/client'; -import { message } from 'antd'; import { saveAs } from 'file-saver'; import { cloneDeep } from 'lodash'; import { useTranslation } from 'react-i18next'; @@ -29,45 +27,43 @@ export const useDownloadXlsxTemplateAction = () => { const apiClient = useAPIClient(); const actionSchema = useFieldSchema(); const compile = useCompile(); - const { getCollectionJoinField } = useCollectionManager(); + const { getCollectionJoinField, getCollectionField } = useCollectionManager(); const { name, title, getField } = useCollection(); const { t } = useTranslation(NAMESPACE); const { schema: importSchema } = useImportSchema(actionSchema); return { async run() { const { importColumns, explain } = cloneDeep(importSchema?.['x-action-settings']?.['importSettings'] ?? {}); - try { - importColumns.forEach((es) => { - const { uiSchema, interface: fieldInterface } = - getCollectionJoinField(`${name}.${es.dataIndex.join('.')}`) ?? {}; - if (isEmpty(uiSchema) && isEmpty(fieldInterface)) { - throw new Error(t('Field {{fieldName}} does not exist', { fieldName: es.dataIndex.join('.') })); + const columns = importColumns + .map((column) => { + const field = getCollectionField(`${name}.${column.dataIndex[0]}`); + if (!field) { + return; } - es.enum = uiSchema?.enum?.map((e) => ({ value: e.value, label: e.label })); - if (!es.enum && uiSchema.type === 'boolean') { - es.enum = [ - { value: true, label: t('Yes') }, - { value: false, label: t('No') }, - ]; + if (field.interface === 'chinaRegion') { + column.dataIndex.push('name'); } - es.defaultTitle = compile(uiSchema?.title); - if (fieldInterface === 'chinaRegion') { - es.dataIndex.push('name'); + column.defaultTitle = compile(field?.uiSchema?.title) || field.name; + if (column.dataIndex.length > 1) { + const subField = getCollectionJoinField(`${name}.${column.dataIndex.join('.')}`); + if (!subField) { + return; + } + column.defaultTitle = column.defaultTitle + '/' + compile(subField?.uiSchema?.title) || subField.name; } - }); - } catch (error) { - message.error(error.message); - return; - } - + return column; + }) + .filter(Boolean); const { data } = await resource.downloadXlsxTemplate( { - title: compile(title), - explain, - columns: JSON.stringify(compile(importColumns)), + values: { + title: compile(title), + explain, + columns: compile(columns), + }, }, { - method: 'get', + method: 'post', responseType: 'blob', }, ); @@ -82,7 +78,7 @@ export const useImportStartAction = () => { const apiClient = useAPIClient(); const actionSchema = useFieldSchema(); const compile = useCompile(); - const { getCollectionJoinField } = useCollectionManager(); + const { getCollectionJoinField, getCollectionField } = useCollectionManager(); const { name, title, getField } = useCollection(); const { t } = useTranslation(NAMESPACE); const { schema: importSchema } = useImportSchema(actionSchema); @@ -92,47 +88,47 @@ export const useImportStartAction = () => { return { async run() { const { importColumns, explain } = cloneDeep(importSchema?.['x-action-settings']?.['importSettings'] ?? {}); - try { - importColumns.forEach((es) => { - const { uiSchema, interface: fieldInterface } = - getCollectionJoinField(`${name}.${es.dataIndex.join('.')}`) ?? {}; - if (isEmpty(uiSchema) && isEmpty(fieldInterface)) { - throw new Error(t('Field {{fieldName}} does not exist', { fieldName: es.dataIndex.join('.') })); + const columns = importColumns + .map((column) => { + const field = getCollectionField(`${name}.${column.dataIndex[0]}`); + if (!field) { + return; } - es.enum = uiSchema?.enum?.map((e) => ({ value: e.value, label: e.label })); - if (!es.enum && uiSchema.type === 'boolean') { - es.enum = [ - { value: true, label: t('Yes') }, - { value: false, label: t('No') }, - ]; + if (field.interface === 'chinaRegion') { + column.dataIndex.push('name'); } - es.defaultTitle = compile(uiSchema?.title); - if (fieldInterface === 'chinaRegion') { - es.dataIndex.push('name'); + column.defaultTitle = compile(field?.uiSchema?.title) || field.name; + if (column.dataIndex.length > 1) { + const subField = getCollectionJoinField(`${name}.${column.dataIndex.join('.')}`); + if (!subField) { + return; + } + column.defaultTitle = column.defaultTitle + '/' + compile(subField?.uiSchema?.title) || subField.name; } - }); - } catch (error) { - message.error(error.message); - return; - } + return column; + }) + .filter(Boolean); let formData = new FormData(); const uploadFiles = form.values.upload.map((f) => f.originFileObj); console.log(form, uploadFiles); formData.append('file', uploadFiles[0]); - formData.append('columns', JSON.stringify(importColumns)); + formData.append('columns', JSON.stringify(columns)); formData.append('explain', explain); setVisible(false); setImportModalVisible(true); setImportStatus(ImportStatus.IMPORTING); - const { data }: any = await apiClient.axios - .post(`${name}:importXlsx`, formData, { + try { + const { data }: any = await apiClient.axios.post(`${name}:importXlsx`, formData, { timeout: 10 * 60 * 1000, - }) - .catch((err) => {}); - setImportResult(data); - form.reset(); - await service?.refresh?.(); - setImportStatus(ImportStatus.IMPORTED); + }); + setImportResult(data); + form.reset(); + await service?.refresh?.(); + setImportStatus(ImportStatus.IMPORTED); + } catch (error) { + setImportModalVisible(false); + setVisible(true); + } }, }; }; diff --git a/packages/plugins/import/src/server/actions/downloadXlsxTemplate.ts b/packages/plugins/import/src/server/actions/downloadXlsxTemplate.ts index f917d4f27..a5141f051 100644 --- a/packages/plugins/import/src/server/actions/downloadXlsxTemplate.ts +++ b/packages/plugins/import/src/server/actions/downloadXlsxTemplate.ts @@ -2,14 +2,14 @@ import { Context, Next } from '@nocobase/actions'; import xlsx from 'node-xlsx'; export async function downloadXlsxTemplate(ctx: Context, next: Next) { - let { columns, explain, title } = ctx.action.params; + let { columns, explain, title } = ctx.request.body; if (typeof columns === 'string') { columns = JSON.parse(columns); } const header = columns?.map((column) => column.defaultTitle); const data = [header]; if (explain?.trim() !== '') { - data.push([explain]); + data.unshift([explain]); } ctx.body = xlsx.build([ diff --git a/packages/plugins/import/src/server/actions/importXlsx.ts b/packages/plugins/import/src/server/actions/importXlsx.ts index 91a45086d..377ec1d13 100644 --- a/packages/plugins/import/src/server/actions/importXlsx.ts +++ b/packages/plugins/import/src/server/actions/importXlsx.ts @@ -1,109 +1,163 @@ import { Context, Next } from '@nocobase/actions'; -import { Repository } from '@nocobase/database'; -import { cloneDeep } from 'lodash'; +import { Collection, Repository } from '@nocobase/database'; import xlsx from 'node-xlsx'; -import { transform } from '../utils'; +import XLSX from 'xlsx'; +import { namespace } from '../../'; const IMPORT_LIMIT_COUNT = 10000; -export async function importXlsx(ctx: Context, next: Next) { - let { columns } = ctx.request.body as any; - const { ['file']: file } = ctx; - const { resourceName, resourceOf } = ctx.action; - if (typeof columns === 'string') { - columns = JSON.parse(columns); - } - const repository = ctx.db.getRepository(resourceName, resourceOf) as Repository; - const collection = repository.collection; +class Importer { + repository: Repository; + collection: Collection; + columns: any[]; + items: any[][] = []; + headerRow; + context: Context; - columns = columns?.filter((col) => col?.dataIndex?.length > 0); - const collectionFields = columns.map((col) => collection.fields.get(col.dataIndex[0])); - const { - 0: { data: originalList }, - } = xlsx.parse(file.buffer); - const failureData = originalList.splice(IMPORT_LIMIT_COUNT + 1); - const titles = originalList.shift(); - const legalList: any[] = []; - if (originalList.length > 0 && titles?.length === columns.length) { - // const results = ( - // await Promise.allSettled( - // originalList.map(async (item) => { - // try { - // const transformResult = await transform({ ctx, record: item, columns, fields: collectionFields }); - // legalList.push(cloneDeep(item)); - // return transformResult; - // } catch (error) { - // failureData.unshift([...item, error.message]); - // } - // }), - // ) - // ).filter((item) => 'value' in item && item.value !== undefined); - const values: any[] = []; - for (const item of originalList) { - try { - const transformResult = await transform({ ctx, record: item, columns, fields: collectionFields }); - values.push(transformResult); - legalList.push(cloneDeep(item)); - } catch (error) { - failureData.unshift([...item, error.message]); + constructor(ctx: Context) { + const { resourceName, resourceOf } = ctx.action; + this.context = ctx; + this.repository = ctx.db.getRepository(resourceName, resourceOf); + this.collection = this.repository.collection; + this.parseXlsx(); + } + + getRows() { + const workbook = XLSX.read(this.context.file.buffer, { + type: 'buffer', + // cellDates: true, + // raw: false, + }); + const r = workbook.Sheets[workbook.SheetNames[0]]; + const rows = XLSX.utils.sheet_to_json(r, { header: 1, defval: null, raw: false }); + return rows; + } + + parseXlsx() { + const rows = this.getRows(); + let columns = this.context.request.body.columns as any[]; + if (typeof columns === 'string') { + columns = JSON.parse(columns); + } + this.columns = columns.map((column) => { + return { + ...column, + field: this.collection.fields.get(column.dataIndex[0]), + }; + }); + const str = this.columns.map((column) => column.defaultTitle).join('||'); + for (const row of rows) { + if (this.hasHeaderRow()) { + if (row && row.join('').trim()) { + this.items.push(row); + } + } + if (str === row.filter((r) => r).join('||')) { + this.headerRow = row; } } - //@ts-ignore - // const values = results.map((r) => r.value); - const result = await ctx.db.sequelize.transaction(async (transaction) => { - let sort: number = 0; - if (collection.options.sortable) { - sort = await repository.model.max('sort', { transaction }); - } - for (const [index, val] of values.entries()) { - if (val === undefined || val === null) { + } + + getFieldByIndex(index) { + return this.columns[index].field; + } + + async getItems() { + const items: any[] = []; + for (const row of this.items) { + const values = {}; + const errors = []; + for (let index = 0; index < row.length; index++) { + if (!this.columns[index]) { continue; } + const column = this.columns[index]; + const { field, defaultTitle } = column; + let value = row[index]; + if (value === undefined || value === null) { + continue; + } + const parser = field.buildValueParser({ ...this.context, column }); + await parser.setValue(typeof value === 'string' ? value.trim() : value); + value = parser.getValue(); + if (parser.errors.length > 0) { + errors.push(`${defaultTitle}: ${parser.errors.join(';')}`); + } + if (value === undefined) { + continue; + } + values[field.name] = value; + } + items.push({ + row, + values, + errors, + }); + } + return items; + } + + hasSortField() { + return !!this.collection.options.sortable; + } + + async run() { + return await this.context.db.sequelize.transaction(async (transaction) => { + let sort: number = 0; + if (this.hasSortField()) { + sort = await this.repository.model.max('sort', { transaction }); + } + const result: any = [[], []]; + for (const { row, values, errors } of await this.getItems()) { + if (errors.length > 0) { + row.push(errors.join(';')); + result[1].push(row); + continue; + } + if (this.hasSortField()) { + values['sort'] = ++sort; + } try { - let values = { ...val }; - if (collection.options.sortable) { - sort += 1; - values['sort'] = sort; - } - await repository.create({ + const instance = await this.repository.create({ values, transaction, logging: false, + context: this.context, }); + result[0].push(instance); } catch (error) { - const failData = legalList[index]; - failData.push(error?.original?.message ?? error.message); - failureData.unshift(failData); + this.context.log.error(error, row); + result[1].push(row); } } - return { - successCount: originalList.length - failureData.length, - failureCount: failureData.length, - }; + return result; }); - const header = columns?.map((column) => column.defaultTitle); - ctx.body = { - rows: xlsx.build([ - { - name: file.originalname, - data: [header].concat(failureData), - }, - ]), - ...result, - }; - } else { - ctx.body = { - rows: file.buffer.toJSON(), - successCount: 0, - failureCount: originalList?.length ?? 0, - }; } - ctx.set({ - 'Content-Type': 'application/octet-stream', - // to avoid "invalid character" error in header (RFC) - 'Content-Disposition': `attachment; filename=${encodeURI('testTitle')}.xlsx`, - }); + hasHeaderRow() { + return !!this.headerRow; + } +} + +export async function importXlsx(ctx: Context, next: Next) { + const importer = new Importer(ctx); + + if (!importer.hasHeaderRow()) { + ctx.throw(400, ctx.t('Imported template does not match, please download again.', { ns: namespace })); + } + + const [success, failure] = await importer.run(); + + ctx.body = { + rows: xlsx.build([ + { + name: ctx.file.originalname, + data: [importer.headerRow].concat(failure), + }, + ]), + successCount: success.length, + failureCount: failure.length, + }; await next(); } diff --git a/packages/plugins/import/src/server/locale/zh-CN.ts b/packages/plugins/import/src/server/locale/zh-CN.ts index a69977251..12b9317e7 100644 --- a/packages/plugins/import/src/server/locale/zh-CN.ts +++ b/packages/plugins/import/src/server/locale/zh-CN.ts @@ -7,4 +7,5 @@ export default { 'Incorrect date format': '日期格式不正确', 'Incorrect email format': '邮箱格式不正确', 'Illegal percentage format': '百分比格式有误', + 'Imported template does not match, please download again.': '导入模板不匹配,请检查导入文件标题行或重新下载导入模板' };