Skip to content

D
dify
Commit d5b321af authored by jyong's avatar jyong
Browse files
Options

Merge remote-tracking branch 'origin/feat/workflow-backend' into feat/workflow-backend

parents f3b46bf7 2008986f
Hide whitespace changes
Inline Side-by-side
Showing with 372 additions and 43 deletions
api/core/workflow/nodes/http_request/entities.py
View file @ d5b321af
from typing import Literal, Union from typing import Literal, Optional, Union
from pydantic import BaseModel from pydantic import BaseModel
...@@ -29,4 +29,4 @@ class HttpRequestNodeData(BaseNodeData): ...@@ -29,4 +29,4 @@ class HttpRequestNodeData(BaseNodeData):
authorization: Authorization authorization: Authorization
headers: str headers: str
params: str params: str
body: Body body: Optional[Body]
\ No newline at end of file \ No newline at end of file
api/core/workflow/nodes/http_request/http_executor.py
View file @ d5b321af
...@@ -43,6 +43,7 @@ class HttpExecutor: ...@@ -43,6 +43,7 @@ class HttpExecutor:
self.params = {} self.params = {}
self.headers = {} self.headers = {}
self.body = None self.body = None
self.files = None
# init template # init template
self._init_template(node_data, variables) self._init_template(node_data, variables)
...@@ -76,11 +77,17 @@ class HttpExecutor: ...@@ -76,11 +77,17 @@ class HttpExecutor:
# fill in params # fill in params
kv_paris = original_params.split('\n') kv_paris = original_params.split('\n')
for kv in kv_paris: for kv in kv_paris:
if not kv.strip():
continue
kv = kv.split(':') kv = kv.split(':')
if len(kv) != 2: if len(kv) == 2:
k, v = kv
elif len(kv) == 1:
k, v = kv[0], ''
else:
raise ValueError(f'Invalid params {kv}') raise ValueError(f'Invalid params {kv}')
k, v = kv
self.params[k] = v self.params[k] = v
# extract all template in headers # extract all template in headers
...@@ -96,51 +103,61 @@ class HttpExecutor: ...@@ -96,51 +103,61 @@ class HttpExecutor:
# fill in headers # fill in headers
kv_paris = original_headers.split('\n') kv_paris = original_headers.split('\n')
for kv in kv_paris: for kv in kv_paris:
if not kv.strip():
continue
kv = kv.split(':') kv = kv.split(':')
if len(kv) != 2: if len(kv) == 2:
k, v = kv
elif len(kv) == 1:
k, v = kv[0], ''
else:
raise ValueError(f'Invalid headers {kv}') raise ValueError(f'Invalid headers {kv}')
k, v = kv
self.headers[k] = v self.headers[k] = v
# extract all template in body # extract all template in body
body_template = re.findall(r'{{(.*?)}}', node_data.body.data or '') or [] if node_data.body:
body_template = list(set(body_template)) body_template = re.findall(r'{{(.*?)}}', node_data.body.data or '') or []
original_body = node_data.body.data or '' body_template = list(set(body_template))
for body in body_template: original_body = node_data.body.data or ''
if not body: for body in body_template:
continue if not body:
continue
original_body = original_body.replace(f'{{{{{body}}}}}', str(variables.get(body, '')))
original_body = original_body.replace(f'{{{{{body}}}}}', str(variables.get(body, ''))) if node_data.body.type == 'json':
self.headers['Content-Type'] = 'application/json'
if node_data.body.type == 'json': elif node_data.body.type == 'x-www-form-urlencoded':
self.headers['Content-Type'] = 'application/json' self.headers['Content-Type'] = 'application/x-www-form-urlencoded'
elif node_data.body.type == 'x-www-form-urlencoded': # elif node_data.body.type == 'form-data':
self.headers['Content-Type'] = 'application/x-www-form-urlencoded' # self.headers['Content-Type'] = 'multipart/form-data'
# elif node_data.body.type == 'form-data':
# self.headers['Content-Type'] = 'multipart/form-data' if node_data.body.type in ['form-data', 'x-www-form-urlencoded']:
body = {}
if node_data.body.type in ['form-data', 'x-www-form-urlencoded']: kv_paris = original_body.split('\n')
body = {} for kv in kv_paris:
kv_paris = original_body.split('\n') kv = kv.split(':')
for kv in kv_paris: if len(kv) == 2:
kv = kv.split(':') body[kv[0]] = kv[1]
if len(kv) != 2: elif len(kv) == 1:
raise ValueError(f'Invalid body {kv}') body[kv[0]] = ''
body[kv[0]] = kv[1] else:
raise ValueError(f'Invalid body {kv}')
if node_data.body.type == 'form-data':
self.files = { if node_data.body.type == 'form-data':
k: ('', v) for k, v in body.items() self.files = {
} k: ('', v) for k, v in body.items()
}
else:
self.body = urlencode(body)
else: else:
self.body = urlencode(body) self.body = original_body
else:
self.body = original_body
def _assembling_headers(self) -> dict[str, Any]: def _assembling_headers(self) -> dict[str, Any]:
authorization = deepcopy(self.authorization) authorization = deepcopy(self.authorization)
headers = deepcopy(self.headers) or [] headers = deepcopy(self.headers) or {}
if self.authorization.type == 'api-key': if self.authorization.type == 'api-key':
if self.authorization.config.api_key is None: if self.authorization.config.api_key is None:
raise ValueError('api_key is required') raise ValueError('api_key is required')
...@@ -232,10 +249,24 @@ class HttpExecutor: ...@@ -232,10 +249,24 @@ class HttpExecutor:
server_url += f'?{urlencode(self.params)}' server_url += f'?{urlencode(self.params)}'
raw_request = f'{self.method.upper()} {server_url} HTTP/1.1\n' raw_request = f'{self.method.upper()} {server_url} HTTP/1.1\n'
for k, v in self.headers.items():
headers = self._assembling_headers()
for k, v in headers.items():
raw_request += f'{k}: {v}\n' raw_request += f'{k}: {v}\n'
raw_request += '\n' raw_request += '\n'
raw_request += self.body or ''
# if files, use multipart/form-data with boundary
if self.files:
boundary = '----WebKitFormBoundary7MA4YWxkTrZu0gW'
raw_request = f'--{boundary}\n' + raw_request
for k, v in self.files.items():
raw_request += f'Content-Disposition: form-data; name="{k}"; filename="{v[0]}"\n'
raw_request += f'Content-Type: {v[1]}\n\n'
raw_request += v[1] + '\n'
raw_request += f'--{boundary}\n'
raw_request += '--\n'
else:
raw_request += self.body or ''
return raw_request return raw_request
\ No newline at end of file
api/core/workflow/nodes/http_request/http_request_node.py
View file @ d5b321af
...@@ -24,15 +24,17 @@ class HttpRequestNode(BaseNode): ...@@ -24,15 +24,17 @@ class HttpRequestNode(BaseNode):
# init http executor # init http executor
try: try:
http_executor = HttpExecutor(node_data=node_data, variables=variables) http_executor = HttpExecutor(node_data=node_data, variables=variables)
# invoke http executor
# invoke http executor
response = http_executor.invoke() response = http_executor.invoke()
except Exception as e: except Exception as e:
return NodeRunResult( return NodeRunResult(
status=WorkflowNodeExecutionStatus.FAILED, status=WorkflowNodeExecutionStatus.FAILED,
inputs=variables, inputs=variables,
error=str(e), error=str(e),
process_data=http_executor.to_raw_request() process_data={
'request': http_executor.to_raw_request()
}
) )
return NodeRunResult( return NodeRunResult(
...@@ -43,7 +45,9 @@ class HttpRequestNode(BaseNode): ...@@ -43,7 +45,9 @@ class HttpRequestNode(BaseNode):
'body': response, 'body': response,
'headers': response.headers 'headers': response.headers
}, },
process_data=http_executor.to_raw_request() process_data={
'request': http_executor.to_raw_request(),
}
) )
......
api/tests/integration_tests/workflow/nodes/__mock/http.py 0 → 100644
View file @ d5b321af
import os
import pytest
import requests.api as requests
import httpx._api as httpx
from requests import Response as RequestsResponse
from httpx import Request as HttpxRequest
from yarl import URL
from typing import Literal
from _pytest.monkeypatch import MonkeyPatch
from json import dumps
MOCK = os.getenv('MOCK_SWITCH', 'false') == 'true'
class MockedHttp:
def requests_request(method: Literal['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], url: str,
**kwargs) -> RequestsResponse:
"""
Mocked requests.request
"""
response = RequestsResponse()
response.url = str(URL(url) % kwargs.get('params', {}))
response.headers = kwargs.get('headers', {})
if url == 'http://404.com':
response.status_code = 404
response._content = b'Not Found'
return response
# get data, files
data = kwargs.get('data', None)
files = kwargs.get('files', None)
if data is not None:
resp = dumps(data).encode('utf-8')
if files is not None:
resp = dumps(files).encode('utf-8')
else:
resp = b'OK'
response.status_code = 200
response._content = resp
return response
def httpx_request(method: Literal['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'],
url: str, **kwargs) -> httpx.Response:
"""
Mocked httpx.request
"""
response = httpx.Response(
status_code=200,
request=HttpxRequest(method, url)
)
response.headers = kwargs.get('headers', {})
if url == 'http://404.com':
response.status_code = 404
response.content = b'Not Found'
return response
# get data, files
data = kwargs.get('data', None)
files = kwargs.get('files', None)
if data is not None:
resp = dumps(data).encode('utf-8')
if files is not None:
resp = dumps(files).encode('utf-8')
else:
resp = b'OK'
response.status_code = 200
response._content = resp
return response
@pytest.fixture
def setup_http_mock(request, monkeypatch: MonkeyPatch):
if not MOCK:
yield
return
monkeypatch.setattr(requests, "request", MockedHttp.requests_request)
monkeypatch.setattr(httpx, "request", MockedHttp.httpx_request)
yield
monkeypatch.undo()
\ No newline at end of file
api/tests/integration_tests/workflow/nodes/test_http.py 0 → 100644
View file @ d5b321af
from calendar import c
import pytest
from core.app.entities.app_invoke_entities import InvokeFrom
from core.workflow.entities.variable_pool import VariablePool
from core.workflow.nodes.http_request.http_request_node import HttpRequestNode
from tests.integration_tests.workflow.nodes.__mock.http import setup_http_mock
BASIC_NODE_DATA = {
'tenant_id': '1',
'app_id': '1',
'workflow_id': '1',
'user_id': '1',
'user_from': InvokeFrom.WEB_APP,
}
# construct variable pool
pool = VariablePool(system_variables={}, user_inputs={})
pool.append_variable(node_id='1', variable_key_list=['123', 'args1'], value=1)
pool.append_variable(node_id='1', variable_key_list=['123', 'args2'], value=2)
@pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True)
def test_get(setup_http_mock):
node = HttpRequestNode(config={
'id': '1',
'data': {
'title': 'http',
'desc': '',
'variables': [{
'variable': 'args1',
'value_selector': ['1', '123', 'args1'],
}],
'method': 'get',
'url': 'http://example.com',
'authorization': {
'type': 'api-key',
'config': {
'type': 'basic',
'api_key':'ak-xxx',
'header': 'api-key',
}
},
'headers': 'X-Header:123',
'params': 'A:b',
'body': None,
}
}, **BASIC_NODE_DATA)
result = node.run(pool)
data = result.process_data.get('request', '')
assert '?A=b' in data
assert 'api-key: Basic ak-xxx' in data
assert 'X-Header: 123' in data
@pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True)
def test_template(setup_http_mock):
node = HttpRequestNode(config={
'id': '1',
'data': {
'title': 'http',
'desc': '',
'variables': [{
'variable': 'args1',
'value_selector': ['1', '123', 'args2'],
}],
'method': 'get',
'url': 'http://example.com/{{args1}}',
'authorization': {
'type': 'api-key',
'config': {
'type': 'basic',
'api_key':'ak-xxx',
'header': 'api-key',
}
},
'headers': 'X-Header:123\nX-Header2:{{args1}}',
'params': 'A:b\nTemplate:{{args1}}',
'body': None,
}
}, **BASIC_NODE_DATA)
result = node.run(pool)
data = result.process_data.get('request', '')
assert '?A=b' in data
assert 'Template=2' in data
assert 'api-key: Basic ak-xxx' in data
assert 'X-Header: 123' in data
assert 'X-Header2: 2' in data
@pytest.mark.parametrize('setup_http_mock', [['none']], indirect=True)
def test_json(setup_http_mock):
node = HttpRequestNode(config={
'id': '1',
'data': {
'title': 'http',
'desc': '',
'variables': [{
'variable': 'args1',
'value_selector': ['1', '123', 'args1'],
}],
'method': 'post',
'url': 'http://example.com',
'authorization': {
'type': 'api-key',
'config': {
'type': 'basic',
'api_key':'ak-xxx',
'header': 'api-key',
}
},
'headers': 'X-Header:123',
'params': 'A:b',
'body': {
'type': 'json',
'data': '{"a": "{{args1}}"}'
},
}
}, **BASIC_NODE_DATA)
result = node.run(pool)
data = result.process_data.get('request', '')
assert '{"a": "1"}' in data
assert 'api-key: Basic ak-xxx' in data
assert 'X-Header: 123' in data
def test_x_www_form_urlencoded(setup_http_mock):
node = HttpRequestNode(config={
'id': '1',
'data': {
'title': 'http',
'desc': '',
'variables': [{
'variable': 'args1',
'value_selector': ['1', '123', 'args1'],
}, {
'variable': 'args2',
'value_selector': ['1', '123', 'args2'],
}],
'method': 'post',
'url': 'http://example.com',
'authorization': {
'type': 'api-key',
'config': {
'type': 'basic',
'api_key':'ak-xxx',
'header': 'api-key',
}
},
'headers': 'X-Header:123',
'params': 'A:b',
'body': {
'type': 'x-www-form-urlencoded',
'data': 'a:{{args1}}\nb:{{args2}}'
},
}
}, **BASIC_NODE_DATA)
result = node.run(pool)
data = result.process_data.get('request', '')
assert 'a=1&b=2' in data
assert 'api-key: Basic ak-xxx' in data
assert 'X-Header: 123' in data
def test_form_data(setup_http_mock):
node = HttpRequestNode(config={
'id': '1',
'data': {
'title': 'http',
'desc': '',
'variables': [{
'variable': 'args1',
'value_selector': ['1', '123', 'args1'],
}, {
'variable': 'args2',
'value_selector': ['1', '123', 'args2'],
}],
'method': 'post',
'url': 'http://example.com',
'authorization': {
'type': 'api-key',
'config': {
'type': 'basic',
'api_key':'ak-xxx',
'header': 'api-key',
}
},
'headers': 'X-Header:123',
'params': 'A:b',
'body': {
'type': 'form-data',
'data': 'a:{{args1}}\nb:{{args2}}'
},
}
}, **BASIC_NODE_DATA)
result = node.run(pool)
data = result.process_data.get('request', '')
assert 'form-data; name="a"' in data
assert '1' in data
assert 'form-data; name="b"' in data
assert '2' in data
assert 'api-key: Basic ak-xxx' in data
assert 'X-Header: 123' in data
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment