Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
D
dify
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
ai-tech
dify
Commits
47b299f5
Commit
47b299f5
authored
Jun 13, 2023
by
jyong
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
add mutil upload batch
parent
0cbb3c15
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
20 additions
and
10 deletions
+20
-10
datasets_document.py
api/controllers/console/datasets/datasets_document.py
+16
-6
document.py
api/controllers/service_api/dataset/document.py
+1
-1
dataset_service.py
api/services/dataset_service.py
+3
-3
No files found.
api/controllers/console/datasets/datasets_document.py
View file @
47b299f5
...
@@ -202,10 +202,15 @@ class DatasetDocumentListApi(Resource):
...
@@ -202,10 +202,15 @@ class DatasetDocumentListApi(Resource):
return
response
return
response
documents_and_batch_fields
=
{
'documents'
:
fields
.
List
(
fields
.
Nested
(
document_fields
)),
'batch'
:
fields
.
String
}
@
setup_required
@
setup_required
@
login_required
@
login_required
@
account_initialization_required
@
account_initialization_required
@
marshal_with
(
document_fields
)
@
marshal_with
(
document
s_and_batch
_fields
)
def
post
(
self
,
dataset_id
):
def
post
(
self
,
dataset_id
):
dataset_id
=
str
(
dataset_id
)
dataset_id
=
str
(
dataset_id
)
...
@@ -239,7 +244,7 @@ class DatasetDocumentListApi(Resource):
...
@@ -239,7 +244,7 @@ class DatasetDocumentListApi(Resource):
DocumentService
.
document_create_args_validate
(
args
)
DocumentService
.
document_create_args_validate
(
args
)
try
:
try
:
documents
=
DocumentService
.
save_document_with_dataset_id
(
dataset
,
args
,
current_user
)
documents
,
batch
=
DocumentService
.
save_document_with_dataset_id
(
dataset
,
args
,
current_user
)
except
ProviderTokenNotInitError
:
except
ProviderTokenNotInitError
:
raise
ProviderNotInitializeError
()
raise
ProviderNotInitializeError
()
except
QuotaExceededError
:
except
QuotaExceededError
:
...
@@ -247,13 +252,17 @@ class DatasetDocumentListApi(Resource):
...
@@ -247,13 +252,17 @@ class DatasetDocumentListApi(Resource):
except
ModelCurrentlyNotSupportError
:
except
ModelCurrentlyNotSupportError
:
raise
ProviderModelCurrentlyNotSupportError
()
raise
ProviderModelCurrentlyNotSupportError
()
return
documents
return
{
'documents'
:
documents
,
'batch'
:
batch
}
class
DatasetInitApi
(
Resource
):
class
DatasetInitApi
(
Resource
):
dataset_and_document_fields
=
{
dataset_and_document_fields
=
{
'dataset'
:
fields
.
Nested
(
dataset_fields
),
'dataset'
:
fields
.
Nested
(
dataset_fields
),
'documents'
:
fields
.
List
(
fields
.
Nested
(
document_fields
))
'documents'
:
fields
.
List
(
fields
.
Nested
(
document_fields
)),
'batch'
:
fields
.
String
}
}
@
setup_required
@
setup_required
...
@@ -276,7 +285,7 @@ class DatasetInitApi(Resource):
...
@@ -276,7 +285,7 @@ class DatasetInitApi(Resource):
DocumentService
.
document_create_args_validate
(
args
)
DocumentService
.
document_create_args_validate
(
args
)
try
:
try
:
dataset
,
documents
=
DocumentService
.
save_document_without_dataset_id
(
dataset
,
documents
,
batch
=
DocumentService
.
save_document_without_dataset_id
(
tenant_id
=
current_user
.
current_tenant_id
,
tenant_id
=
current_user
.
current_tenant_id
,
document_data
=
args
,
document_data
=
args
,
account
=
current_user
account
=
current_user
...
@@ -290,7 +299,8 @@ class DatasetInitApi(Resource):
...
@@ -290,7 +299,8 @@ class DatasetInitApi(Resource):
response
=
{
response
=
{
'dataset'
:
dataset
,
'dataset'
:
dataset
,
'documents'
:
documents
'documents'
:
documents
,
'batch'
:
batch
}
}
return
response
return
response
...
...
api/controllers/service_api/dataset/document.py
View file @
47b299f5
...
@@ -78,7 +78,7 @@ class DocumentListApi(DatasetApiResource):
...
@@ -78,7 +78,7 @@ class DocumentListApi(DatasetApiResource):
}
}
try
:
try
:
documents
=
DocumentService
.
save_document_with_dataset_id
(
documents
,
batch
=
DocumentService
.
save_document_with_dataset_id
(
dataset
=
dataset
,
dataset
=
dataset
,
document_data
=
document_data
,
document_data
=
document_data
,
account
=
dataset
.
created_by_account
,
account
=
dataset
.
created_by_account
,
...
...
api/services/dataset_service.py
View file @
47b299f5
...
@@ -487,7 +487,7 @@ class DocumentService:
...
@@ -487,7 +487,7 @@ class DocumentService:
# trigger async task
# trigger async task
document_indexing_task
.
delay
(
dataset
.
id
,
document_ids
)
document_indexing_task
.
delay
(
dataset
.
id
,
document_ids
)
return
documents
return
documents
,
batch
@
staticmethod
@
staticmethod
def
save_document
(
dataset
:
Dataset
,
process_rule_id
:
str
,
data_source_type
:
str
,
data_source_info
:
dict
,
def
save_document
(
dataset
:
Dataset
,
process_rule_id
:
str
,
data_source_type
:
str
,
data_source_info
:
dict
,
...
@@ -611,7 +611,7 @@ class DocumentService:
...
@@ -611,7 +611,7 @@ class DocumentService:
db
.
session
.
add
(
dataset
)
db
.
session
.
add
(
dataset
)
db
.
session
.
flush
()
db
.
session
.
flush
()
documents
=
DocumentService
.
save_document_with_dataset_id
(
dataset
,
document_data
,
account
)
documents
,
batch
=
DocumentService
.
save_document_with_dataset_id
(
dataset
,
document_data
,
account
)
cut_length
=
18
cut_length
=
18
cut_name
=
documents
[
0
]
.
name
[:
cut_length
]
cut_name
=
documents
[
0
]
.
name
[:
cut_length
]
...
@@ -619,7 +619,7 @@ class DocumentService:
...
@@ -619,7 +619,7 @@ class DocumentService:
dataset
.
description
=
'useful for when you want to answer queries about the '
+
documents
[
0
]
.
name
dataset
.
description
=
'useful for when you want to answer queries about the '
+
documents
[
0
]
.
name
db
.
session
.
commit
()
db
.
session
.
commit
()
return
dataset
,
documents
return
dataset
,
documents
,
batch
@
classmethod
@
classmethod
def
document_create_args_validate
(
cls
,
args
:
dict
):
def
document_create_args_validate
(
cls
,
args
:
dict
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment