Clusters

clickhouse/migrations/schema_43_request_response_rmt_embedding.sql

@@ -0,0 +1,3 @@
+ALTER TABLE request_response_rmt
+ADD COLUMN embedding Array(Float32)
+AFTER request_body;

clusters/app.py

@@ -0,0 +1,42 @@
+from fastapi import FastAPI
+from pydantic import BaseModel
+
+import pandas as pd
+import numpy as np
+import hdbscan
+from umap import UMAP
+
+app = FastAPI()
+
+
+class RequestWithEmbedding(BaseModel):
+    request_id: str
+    embedding: list[float]
+    signed_body_url: str
+
+
+@app.post("/clusters")
+async def generate_clusters(requests: list[RequestWithEmbedding]):
+    print(requests)
+    df = pd.DataFrame(requests, columns=[
+                      "request_id", "embedding", "signed_body_url"])
+    embeddings = [np.array(x.embedding) for x in requests]
+
+    hdb = hdbscan.HDBSCAN(min_samples=2, min_cluster_size=2).fit(embeddings)
+
+    umap = UMAP(n_components=2, random_state=42, n_neighbors=80, min_dist=0.1)
+
+    df_umap = (
+        pd.DataFrame(umap.fit_transform(
+            np.array(embeddings)), columns=['x', 'y'])
+        .assign(cluster=lambda df: hdb.labels_.astype(str))
+        # .query('cluster != "-1"')
+        .sort_values(by='cluster')
+    )
+
+    df["cluster"] = hdb.labels_.astype(str)
+
+    # Add the request_body to df_umap
+    df_umap["request_id"] = df["request_id"].apply(lambda x: x[1])
+    df_umap["signed_body_url"] = df["signed_body_url"].apply(lambda x: x[1])
+    return df_umap.to_dict(orient="records")

clusters/generate_clusters.py

@@ -0,0 +1,65 @@
+# import clickhouse_connect
+# from openai import OpenAI
+
+# import pandas as pd
+# import numpy as np
+# import hdbscan
+# from umap import UMAP
+# import plotly.express as px
+
+# client = clickhouse_connect.get_client(
+#     host="localhost", port=18123, username="default", password="")
+
+# rows_with_embeddings = client.query(
+#     "SELECT request_body,embedding FROM request_response_rmt WHERE notEmpty(embedding)").result_rows
+
+# df = pd.DataFrame(rows_with_embeddings, columns=[
+#     "request_body", "embedding"])
+# embeddings = [np.array(x[1]) for x in rows_with_embeddings]
+
+# hdb = hdbscan.HDBSCAN(min_samples=2, min_cluster_size=2).fit(embeddings)
+
+# umap = UMAP(n_components=2, random_state=42, n_neighbors=80, min_dist=0.1)
+
+# df_umap = (
+#     pd.DataFrame(umap.fit_transform(
+#         np.array(embeddings)), columns=['x', 'y'])
+#     .assign(cluster=lambda df: hdb.labels_.astype(str))
+#     # .query('cluster != "-1"')
+#     .sort_values(by='cluster')
+# )
+
+# df["cluster"] = hdb.labels_.astype(str)
+
+# # Add the request_body to df_umap
+# df_umap["request_body"] = df["request_body"]
+
+# # Update the Plotly figure creation
+# fig = px.scatter(
+#     df_umap,
+#     x='x',
+#     y='y',
+#     color='cluster',
+#     hover_data=['request_body']  # Add request_body to hover information
+# )
+
+# # Customize hover template to show only request_body
+# fig.update_traces(
+#     hovertemplate="<br>".join([
+#         "Request Body: %{customdata[0]}",
+#     ])
+# )
+
+# fig.show()
+
+# # for c in df.cluster.unique():
+# #     print(c)
+# #     print(df[df.cluster == c]["request_body"].tolist())
+
+import clickhouse_connect
+
+client = clickhouse_connect.get_client(
+    host="localhost", port=18123, username="default", password="")
+
+client.command(
+    "ALTER TABLE request_response_rmt ADD COLUMN embedding Array(Float32) AFTER request_body")

clusters/requirements.txt

@@ -0,0 +1,126 @@
+annotated-types==0.7.0
+anyio==4.6.2.post1
+appnope==0.1.4
+argon2-cffi==23.1.0
+argon2-cffi-bindings==21.2.0
+arrow==1.3.0
+asttokens==2.4.1
+async-lru==2.0.4
+attrs==24.2.0
+babel==2.16.0
+beautifulsoup4==4.12.3
+bleach==6.1.0
+certifi==2024.8.30
+cffi==1.17.1
+charset-normalizer==3.4.0
+click==8.1.7
+clickhouse-connect==0.8.3
+comm==0.2.2
+debugpy==1.8.7
+decorator==5.1.1
+defusedxml==0.7.1
+distro==1.9.0
+executing==2.1.0
+fastjsonschema==2.20.0
+fqdn==1.5.1
+h11==0.14.0
+hdbscan==0.8.39
+httpcore==1.0.6
+httpx==0.27.2
+idna==3.10
+ipykernel==6.29.5
+ipython==8.28.0
+isoduration==20.11.0
+jedi==0.19.1
+Jinja2==3.1.4
+jiter==0.6.1
+joblib==1.4.2
+json5==0.9.25
+jsonpointer==3.0.0
+jsonschema==4.23.0
+jsonschema-specifications==2024.10.1
+jupyter-events==0.10.0
+jupyter-lsp==2.2.5
+jupyter_client==8.6.3
+jupyter_core==5.7.2
+jupyter_server==2.14.2
+jupyter_server_terminals==0.5.3
+jupyterlab==4.2.5
+jupyterlab_pygments==0.3.0
+jupyterlab_server==2.27.3
+llvmlite==0.43.0
+lz4==4.3.3
+MarkupSafe==3.0.2
+matplotlib-inline==0.1.7
+minio==7.2.9
+mistune==3.0.2
+mypy-extensions==1.0.0
+nbclient==0.10.0
+nbconvert==7.16.4
+nbformat==5.10.4
+nest-asyncio==1.6.0
+notebook==7.2.2
+notebook_shim==0.2.4
+numba==0.60.0
+numpy==2.0.2
+openai==1.51.2
+overrides==7.7.0
+packaging==24.1
+pandas==2.2.3
+pandocfilters==1.5.1
+parso==0.8.4
+pathspec==0.12.1
+pexpect==4.9.0
+platformdirs==4.3.6
+plotly==5.24.1
+prometheus_client==0.21.0
+prompt_toolkit==3.0.48
+psutil==6.1.0
+ptyprocess==0.7.0
+pure_eval==0.2.3
+pycparser==2.22
+pycryptodome==3.20.0
+pydantic==2.9.2
+pydantic_core==2.23.4
+Pygments==2.18.0
+pynndescent==0.5.13
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-json-logger==2.0.7
+pytz==2024.2
+PyYAML==6.0.2
+pyzmq==26.2.0
+referencing==0.35.1
+regex==2024.9.11
+requests==2.32.3
+rfc3339-validator==0.1.4
+rfc3986-validator==0.1.1
+rpds-py==0.20.0
+scikit-learn==1.5.2
+scipy==1.14.1
+Send2Trash==1.8.3
+setuptools==75.2.0
+six==1.16.0
+sniffio==1.3.1
+soupsieve==2.6
+stack-data==0.6.3
+tabulate==0.9.0
+tenacity==9.0.0
+terminado==0.18.1
+threadpoolctl==3.5.0
+tiktoken==0.8.0
+tinycss2==1.3.0
+tornado==6.4.1
+tqdm==4.66.5
+traitlets==5.14.3
+types-python-dateutil==2.9.0.20241003
+typing_extensions==4.12.2
+tzdata==2024.2
+umap-learn==0.5.6
+uri-template==1.3.0
+urllib3==2.2.3
+wcwidth==0.2.13
+webcolors==24.8.0
+webencodings==0.5.1
+websocket-client==1.8.0
+zstandard==0.23.0