# migrations

dir: embeddings_queue
filename: 00001-embeddings.sqlite.sql
hash: d3755dfd232be8e8301f4d7fcfb3a486
sql:
CREATE TABLE embeddings_queue (
    seq_id INTEGER PRIMARY KEY,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    operation INTEGER NOT NULL,
    topic TEXT NOT NULL,
    id TEXT NOT NULL,
    vector BLOB,
    encoding TEXT,
    metadata TEXT
);


dir: sysdb
filename: 00001-collections.sqlite.sql
hash: 38352d725ad1c16074fac420b22b4633
sql:
CREATE TABLE collections (
    id TEXT PRIMARY KEY,
    name TEXT NOT NULL,
    topic TEXT NOT NULL,
    UNIQUE (name)
);

CREATE TABLE collection_metadata (
    collection_id TEXT REFERENCES collections(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (collection_id, key)
);


dir: sysdb
filename: 00002-segments.sqlite.sql
hash: 2913cb6a503055a95f625448037e8912
sql:
CREATE TABLE segments (
    id TEXT PRIMARY KEY,
    type TEXT NOT NULL,
    scope TEXT NOT NULL,
    topic TEXT,
    collection TEXT REFERENCES collection(id)
);

CREATE TABLE segment_metadata (
    segment_id TEXT  REFERENCES segments(id) ON DELETE CASCADE,
    key TEXT NOT NULL,
    str_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (segment_id, key)
);


dir: sysdb
filename: 00004-tenants-databases.sqlite.sql
hash: 048867ce8fcdefe4023c7110e4433591
sql:
CREATE TABLE IF NOT EXISTS tenants (
    id TEXT PRIMARY KEY,
    UNIQUE (id)
);

CREATE TABLE IF NOT EXISTS databases (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per tenant
    tenant_id TEXT NOT NULL REFERENCES tenants(id) ON DELETE CASCADE,
    UNIQUE (tenant_id, name) -- Ensure that a tenant has only one database with a given name
);

CREATE TABLE IF NOT EXISTS collections_tmp (
    id TEXT PRIMARY KEY, -- unique globally
    name TEXT NOT NULL, -- unique per database
    topic TEXT NOT NULL,
    dimension INTEGER,
    database_id TEXT NOT NULL REFERENCES databases(id) ON DELETE CASCADE,
    UNIQUE (name, database_id)
);

-- Create default tenant and database
INSERT OR REPLACE INTO tenants (id) VALUES ('default_tenant'); -- The default tenant id is 'default_tenant' others are UUIDs
INSERT OR REPLACE INTO databases (id, name, tenant_id) VALUES ('00000000-0000-0000-0000-000000000000', 'default_database', 'default_tenant');

INSERT OR REPLACE INTO collections_tmp (id, name, topic, dimension, database_id)
    SELECT id, name, topic, dimension, '00000000-0000-0000-0000-000000000000' FROM collections;
DROP TABLE collections;
ALTER TABLE collections_tmp RENAME TO collections;


dir: metadb
filename: 00001-embedding-metadata.sqlite.sql
hash: 2b4cf52c4bb2676e21d6860a4409f856
sql:
CREATE TABLE embeddings (
    id INTEGER PRIMARY KEY,
    segment_id TEXT NOT NULL,
    embedding_id TEXT NOT NULL,
    seq_id BLOB NOT NULL,
    created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
    UNIQUE (segment_id, embedding_id)
);

CREATE TABLE embedding_metadata (
    id INTEGER REFERENCES embeddings(id),
    key TEXT NOT NULL,
    string_value TEXT,
    int_value INTEGER,
    float_value REAL,
    PRIMARY KEY (id, key)
);

CREATE TABLE max_seq_id (
    segment_id TEXT PRIMARY KEY,
    seq_id BLOB NOT NULL
);

CREATE VIRTUAL TABLE embedding_fulltext USING fts5(id, string_value);


dir: metadb
filename: 00002-embedding-metadata.sqlite.sql
hash: 12a570f7121b3a8ce750a2a7c36da20f
sql:
-- SQLite does not support adding check with alter table, as a result, adding a check
-- involve creating a new table and copying the data over. It is over kill with adding
-- a boolean type column. The application write to the table needs to ensure the data
-- integrity.
ALTER TABLE embedding_metadata ADD COLUMN bool_value INTEGER


dir: metadb
filename: 00003-full-text-tokenize.sqlite.sql
hash: f97ad6334aeaa8f419f01110b648b97a
sql:
CREATE VIRTUAL TABLE embedding_fulltext_search USING fts5(string_value, tokenize='trigram');
INSERT INTO embedding_fulltext_search (rowid, string_value) SELECT rowid, string_value FROM embedding_metadata;
DROP TABLE embedding_fulltext;


# embeddings_queue

created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 7049e1dc-1869-4ea5-b89b-e87594f21b57
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "The Artificial Intelligence (AI) industry has seen unprecedented growth in recent years, reshaping how we interact with technology. The growth rate accelerated in 2023, with the release of human-like conversation chatbots like ChatGPT, and the plethora of subsequent technologies based on Large Language Models (LLMs) and generative algorithms. As a consequence, the industry is in the midst of an AI-driven revolution, which has the potential to lead the way for future-shaping business opportunities."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 0266bc64-e48d-40ac-a071-f6604b31e72e
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "The fuel powering all of this AI machinery is data. However, data is only useful if properly processed and integrated into robust and scalable systems. In other words, data processing infrastructures constitute the foundations of high-quality AI products, and mastering the Data Engineering skills needed for their design and implementation holds paramount value."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: c1db25f6-cecc-4950-9a55-b1be86f3eb1a
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "This lesson has been ingrained in our practice at Tryolabs, borne out of more than 10 years of dedicated work in AI and advanced analytics consulting. Crafting bespoke Data Engineering strategies stands at the heart of our solutions, embodying the essential expertise we deliver to our clients. We've not only navigated the complexities of data-driven challenges but also unlocked tailored, innovative solutions across a diverse range of industries."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d4deb504-ebec-4885-aca2-8bcc33b4bc28
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "In this blog post, we are sharing the essential technical skills but more so the strategic approaches required for crafting scalable and robust data infrastructures. We're doing so through the lens of two projects undertaken by Tryolabs. These examples serve to illustrate how theory and practice intertwine, offering a comprehensive look at the challenges and solutions encountered in our quest to leverage the power of AI.\n\nExhibit 1: Web traffic analytics for SEO scaling"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d343785b-f39b-4c7f-b0e7-d2f043d44dc3
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Exhibit 1: Web traffic analytics for SEO scaling\n\nBack in 2018, this project marked a pivotal moment in our venture into the realm of large-scale Data Engineering. The mission was to architect and deploy a data infrastructure capable of supporting high-volume data reports on web traffic for a leading client in the digital marketing space.\n\nNavigating high-volume data for strategic SEO"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 404d1903-4c45-404a-8e2f-699431fb62df
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Navigating high-volume data for strategic SEO\n\nWe embarked on developing an analytics tool for Search Engine Optimization (SEO), a core component of our client's digital marketing services. The goal was to empower web users\u2014whether individuals or businesses\u2014 to boost their online visibility, drawing significantly more traffic by refining their web content strategies."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: fddc899e-2aaa-4c9e-914e-369998f9cfb4
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Faced with the need to process vast volumes of information, our task was to distill actionable data products from the ocean of online traffic data\u2014covering keywords, pages, and domains search frequencies. This wasn't just about handling data; it was about unlocking strategies for SEO success by understanding what makes content appealing to search engines and potential visitors.\n\nFigure 1.1. Illustration of the SEO tool for web traffic analytics"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: dcefce6e-0e56-4b45-aad8-56288976dc6c
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Figure 1.1. Illustration of the SEO tool for web traffic analytics\n\nThe Data Engineering challenge To get an idea of the required data processing scale, close to 100 million keywords needed to be processed every day, and monthly aggregations required processing over two billion keywords in less than 48 hours every month.\n\nTackling this challenge was an ambitious task that demanded a robust, efficient, and scalable data management strategy.\n\nCrafting the solution: Data Engineering in action\n\nOur solution to this challenge was a meticulously planned data processing strategy that encompassed the entire data lifecycle, from acquisition to actionable data products. The approach was threefold:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: bc449fcf-0356-4dab-89bc-171fd1045b3c
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Data centralization. Data processing starts by gathering raw traffic data, primarily from Google's SERP, into a unified storage place, setting the stage for all subsequent data transformations. Extraction and processing. With raw data centralized, a carefully crafted data process is put in place to convert crude data into ripe data products relevant to the SEO tool. Storage and access. The final piece of the puzzle was storing this processed data in a way that made it easily accessible for analysis and reporting. This meant not only choosing the right storage solutions but also making design choices ensuring fast, efficient queries."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 44a164d9-816f-4b5e-ade2-6abd6779db6f
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Using the Data Engineering lexicon, our infrastructure utilized a combination of data laking, ETL pipelines, and data warehousing (points 1, 2, and 3 above, respectively), ensuring seamless access to data products.\n\nData Engineering components of the solution Data laking - gathering files containing Google generated data (e.g. SERP) about any given keyword.\n\nETL pipeline - extract keyword and associated SERP data from the raw gathered files, transform the data to a custom format and load to various databases."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 744bae62-a975-4663-8616-b63342d69298
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Data warehousing - the ETL pipeline produces different types of processed data depending on the querying complexity of the required analytical reports, calling for different warehousing solutions.\n\nTo bring this vision to life, we leveraged the power and flexibility of Amazon Web Services (AWS). The AWS ecosystem offers a suite of managed and serverless products aligned with our need for scalability and efficiency. This choice allowed us to focus on optimizing our Data Engineering processes without being bogged down by infrastructure management."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d75b8758-5d04-487e-a679-9e79c67c4096
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Architecture design key decisions Requirements Provision of raw data (batches) by 3rd party should trigger data processing. Data processing should be easily scalable. Data products should support both simple and complex queries as well as expensive aggregations. Design decisions AWS S3 bucket for raw data storage with event triggers feeding an SQS Queue. ECS-based data processing triggered by listening to the SQS Queue. Warehousing: DynamoDB and Elasticsearch (plus aggregations using AWS Batch)."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 1482aed2-b814-4003-8970-a226459a28aa
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "This diagram illustrates the architecture of the implemented data infrastructure. It highlights the specific AWS services employed at each step of the Data Engineering process and the rationale behind our architectural choices.\n\nFigure 1.2. Scheme of the data infrastructure design indicating the AWS products/services used, highlighting their corresponding Data Engineering task.\n\nKey architectural insights\n\nDelving into the architecture illustrated in Figure 1.2 provides deeper details about the inner workings of the solution, showcasing how each component contributes to an efficient data infrastructure.\n\nData laking"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: bce51407-feeb-4db8-8f2e-79a2a96275ff
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Data laking\n\nAn S3 bucket was used as data lake, where raw data files, each containing 100 Google search results for a given keyword (along its metadata), were securely stored upon upload by a third-party provider. The latter triggered an S3 event, queueing a message in an SQS queue, readying the system for the next data processing phase.\n\nETL pipelines\n\nThe ETL pipelines were designed with a dual approach to cater event-based and batch data processing needs.\n\nEvent-based custom pipeline"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 87e48084-ef12-473d-a37d-8e526003b6d2
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "The ETL pipelines were designed with a dual approach to cater event-based and batch data processing needs.\n\nEvent-based custom pipeline\n\nAn ECS cluster hosts a docker microservice that continuously listens to the SQS queue. Upon new messages arrival, it downloads the new raw data file from the S3 bucket, process the keyword-related data, and loads it, after custom formatting, into DynamoDB and Elasticsearch.\n\nBatch (monthly) aggregation pipelines\n\nA first AWS Batch Job is used to process the DynamoDB table pages_data to obtain aggregations at page level, writing into the DynamoDB table pages_aggregated , useful for page-level analytics reports."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 19f24217-2975-488b-a45e-a174c60ada72
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "A second AWS Batch Job is used to process the DynamoDB table pages_aggregated (first batch job) to obtain aggregations at domain level, writing into the DynamoDB table domains_aggregated , useful for domain-level analytics reports.\n\nData warehousing\n\nThe event-based ETL pipeline loads transformed data to two types of storage services:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 831944d4-77c6-4e18-9715-eecee07460a6
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Data warehousing\n\nThe event-based ETL pipeline loads transformed data to two types of storage services:\n\nDynamoDB keywords stores keyword metadata (simple queries) pages_data a huge table partitioned by domain:page and sorted by keyword, that will be used downstream for additional processing (page/domain level aggregations) Elasticsearch keywords stores keyword metadata (complex queries) SERP stores the SERP data for each keyword Elasticsearch storage resources were needed to process queries of high complexity not possible with DynamoDB."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 3569c2be-c7e3-4742-8c28-eb323987890a
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "The first AWS Batch Job aggregates data at page level and loads the output to DynamoDB table pages_aggregated which, being partitioned by domain and sorted by website path, makes access to page level aggregated data more efficient and cost effective.\n\nThe second AWS Batch Job aggregates data at domain level and loads the output to DynamoDB table domains_aggregated which, being partitioned by domain and sorted by month, makes access to domain level aggregated data more efficient and cost effective.\n\nThis data infrastructure allowed for the scalable generation of a set of key data products that supported the analytical reports required by the client\u2019s SEO tool (see Figure 1.3)."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 57bb3b9b-dadc-4cca-896a-26ec3ad4f6e5
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Key data products Elasticsearch tables SERP & keywords : support complex keyword based queries (e.g. top keywords) and complex text searches (e.g. content leads, which are the top ranking pages for a given keyword). DynamoDB tables keywords : supports fast and scalable basic reports on keywords related data pages_aggregated and domains_aggregated : support fast and scalable reports on domain and website traffic evolution.\n\nFigure 1.3. Usage of the main data products in a diversity of analytics reports of the SEO tool.\n\nTakeaways"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: e1c11029-7a5b-4558-80d2-381b0d3aec39
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Figure 1.3. Usage of the main data products in a diversity of analytics reports of the SEO tool.\n\nTakeaways\n\nThe implemented infrastructure provided the necessary data products to fulfill the analytics needs of the client, achieving the required throughput (e.g. over 2 billion keywords being processed in less than 48 hrs every month). This was a consequence of appropriate design decisions, some of which are worth highlighting.\n\nEfficient partitioning and sorting keys"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: dd98767b-de57-4021-adfc-fc59c514d7f4
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Efficient partitioning and sorting keys\n\nAppropriate design of the partitioning and sorting keys in DynamoDB tables played a central role in minimizing table scans, consequently reducing querying costs. This resulted in money-wise efficient data products that were able to support the analytics and reporting needs of the client\u2019s SEO tool.\n\nDeep understanding of DynamoDB\n\nDetailed knowledge of not only partitioning and sorting keys, but also of read and write capacity units and pricing characteristics allowed for appropriate decisions on monthly quote reservations, and table writing mode (only once per record), resulting in performance maximization while keeping low costs."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 065d8f0c-6634-432f-8673-e9d1540c6bd5
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Choosing the appropriate data warehouse\n\nAn appropriate combination of DynamoDB and Elasticsearch allowed for serving data products supporting different querying complexity levels, providing the required data processing flexibility in an affordable manner.\n\nChoosing the appropriate computational scaling method"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: c8506231-7fbb-4a07-86d8-10ba970dc11e
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Choosing the appropriate computational scaling method\n\nAt the beginning of the project, Spark was an appealing option for data processing parallelization, allowing to leverage a rich environment and set of tools, and reducing maintainability (and future scalability issues). However, (in our hands) a custom parallelization approach resulted in higher throughput (probably a Spark initialization overhead issue). Although this should be revisited if working on a similar project nowadays (remember this project took place six years ago), it is a relevant learning to properly weight the available computational scaling methods, before blindly jumping into mainstream tools."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: feb7d0b2-767d-485d-96b7-1c0954fb5597
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Exhibit 2: Enhancing online retail with image semantic search\n\nAs part of a long-lasting engagement with one of our clients, this project constituted a capstone in our journey towards integrating Data Engineering skills and AI expertise.\n\nThe challenge was to build a semantic search service for retail items, providing a robust data infrastructure capable of supporting various Machine Learning applications designed to tackle distinct business challenges.\n\nDeveloping a unified semantic search engine"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: bc4758f3-75e6-49d1-ad39-3fd25c0480e1
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Developing a unified semantic search engine\n\nOur mission was to create an image-based semantic search service that could accurately and efficiently identify items similar to a given target, utilizing both image data and item metadata as inputs. This service was envisioned as a central hub, offering support to a suite of ML-driven products, including recommendation systems, dynamic pricing models, and automatic attribute tagging tools. These products work in synergy to address a broad spectrum of business needs (as illustrated in Figure 2.1)."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 8809148a-27fa-403b-890a-70c56d0c79aa
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "The challenge extended beyond the mere development of these services; it involved ensuring the system could handle the high volume of semantic search queries triggered by user interactions across the ML products. Addressing this required a focused effort on building scalable, real-time data processing capabilities. This endeavor was critical not only for enabling efficient semantic (and keyword-based) searches but also for optimizing the processing of retail items. The outcome was a system that could support timely and effective online product listings, significantly enhancing the retail experience and profitability."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: fea8ebe4-3509-4816-9241-09d4582b90ac
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Figure 2.1. Illustration of the image semantic search service providing the data infrastructure for multiple Machine Learning based products. In this fictitious example, a user requests a business task that requires finding items similar to a floppy disk.\n\nThe Data Engineering challenge Supporting a semantic search service for retail item retrieval, covering over 3.5 million items, and handling close to 80 requests per second, from multiple business processes triggered by an end user that expects a latency below 500 milliseconds.\n\nFulfilling this endeavor was a challenging task requiring cloud orchestration savviness and proper integration of AI experience into data architecture design."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 9a1f37e2-e2fc-446b-89f9-2a5ce669343a
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Building the solution: an AI and Data Engineering mix\n\nThe approach we adopted to solve this challenge was a carefully crafted data operation that transformed items metadata from multiple concurrent sources and items images, into entries of a vector database (the core data product) that efficiently serves hybrid keyword/semantic queries. This solution contained four main components:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 8881378c-e9f8-4155-9ae1-300a9fe1f6fb
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Data centralization The first data processing step is gathering all retail item images in a centralized storage location (supporting unstructured data), which works as the reference source for subsequent transformations resulting in vector encodings indexed in the vector database. Streaming and batch processing Once images are centrally stored, they are extracted and transformed into vector encodings using neural nets and custom transformations. This is done both in batch and streaming modes. Both modes are also used for items metadata processing, carefully considering a key requirement: the need to integrate multiple concurrent metadata sources. Storage supporting semantic retrieval The"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 22c30a5b-85a8-454b-9069-e27498552118
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "carefully considering a key requirement: the need to integrate multiple concurrent metadata sources. Storage supporting semantic retrieval The final step of this data operation is storing the items metadata and vector encoding in a vector database, providing efficient and fast hybrid keyword and semantic retrieval. Monitoring With the main components of the data infrastructure up and running, it is critical to ensure adequate performance levels are continuously delivered. To that end, a monitoring service was put in place to track and fire alerts about the health of the vector database."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 7aadd02c-1ee7-4d21-a5ab-1565b22401bd
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "In Data Engineering terms, this approach required the orchestration of data laking, neural network inference integrated into streaming and batch ETL pipelines, data warehousing, and data product health monitoring (points 1, 2, 3, and 4 above, respectively), resulting in a robust data operation.\n\nData Engineering components of the solution Data laking : gathering raw input images of all retail items.\n\nBatch ETL pipeline : items metadata processing, and neural net vector encoding of item images extracted from the data lake.\n\nStreaming ETL pipeline : near real-time processing of multiple source items metadata, and neural net vector encoding of new item images added to the data lake."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 0354c5ca-6d63-4f46-9872-14cc10e18f40
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Data warehousing : storing ETL pipelines products \u2014 processed items metadata and vector encodings \u2014 into a vector database supporting hybrid keyword-semantic search and approximate kNN methods.\n\nMonitoring: keeping track of the data warehouse health to ensure the items retrieval service fulfills the performance requirements.\n\nTo construct this architecture we combined the AWS, the Google Cloud Platform (GCP), and third-party tools. This allowed us to take advantage of the different strengths offered by such platforms and services, while fulfilling client-specific requirements."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: a975a3ba-7da3-48e8-8e20-58babb93ef5c
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Architecture design key decisions Requirements Integrates (near real-time) items metadata coming from different divisions in the client Supports hybrid keyword-semantic search queries Low latency serving to multiple internal ML-products Design decisions AWS S3 bucket for images laking, and a Kafka server for multi-source real-time data processing GCP Dataflow for batch pipeline, and a combination of Cloud Scheduler and Cloud Run for streaming pipeline Warehousing: Elasticsearch (items metadata and image vector encoding) and BigQuery (items metadata)"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 940adac9-ccfa-48c9-8c93-427b3e65d89f
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Here you can see the architecture of the implemented data operation, indicating the particular platform-dependent services used in each piece of the Data Engineering puzzle, along the principles underlying the chosen design.\n\nFigure 2.2. Scheme of the data infrastructure design indicating the AWS, GCP and third-party services used, highlighting their corresponding Data Engineering task.\n\nKey architectural insights\n\nDigging into the architecture illustrated in Figure 2.2 sheds light on the features of the implemented solution in finer detail, depicting the interplay between each component, which resulted in a robust data infrastructure.\n\nData laking"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 2faf95f0-cfec-4015-8f7f-35a2a4994482
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Data laking\n\nAn AWS S3 bucket was used as data lake to store all retail item images, providing a secure and consistent reference point for data extraction by streaming and batch pipelines.\n\nETL pipelines"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 3dce2824-bf9e-4fc1-b68e-e47dfd90d1a6
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "ETL pipelines\n\nAn on-demand batch pipeline, implemented using GCP Dataflow (Apache Beam), extracts items images from the AWS S3 bucket, loads a cost effective resnet-50 convolutional neural net (stored on GCP Cloud storage), and uses the latter plus custom normalization (enhances similarity distance accuracy), to generate vector encodings from the images. Items metadata and vector encodings are pushed to an Elasticsearch vector database. In addition, items metadata is also stored in BigQuery to facilitate custom analytics and Elasticsearch backfilling."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6453806d-42c9-4cd8-a894-3ee4be1f01fd
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "A streaming pipeline receives items metadata messages published by different divisions within the client to a Kafka server, which is connected to Cloud Run (triggered using Cloud Scheduler), for near real-time data processing. The latter includes i) the addition of new items, applying the same transformations and storage as used in the batch pipeline, and ii) the continuous integration of items metadata updates published by different teams in the client.\n\nData warehousing"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 83d23b71-faa1-4efe-a148-d3c985495acb
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Data warehousing\n\nA resnet-50 convolutional neural net is stored in a GCP Cloud Storage bucket. This model is used to generate image vector encodings supporting semantic search. Items metadata is stored in GCP BigQuery to support custom analytics and Elasticsearch backfilling. Items metadata and image vector encodings are stored in an Elasticsearch vector database supporting hybrid keyword-semantic search, and using the Hierarchical Navigable Small World (HNSW) approximate kNN method for efficient similarity-based retrieval.\n\nMonitoring\n\nThe status of the vector database is monitored using Datadog, keeping track, for example, of index rates and search rates."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 40628d4d-9664-4150-a9af-647234baa506
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Monitoring\n\nThe status of the vector database is monitored using Datadog, keeping track, for example, of index rates and search rates.\n\nThis system robustly supports the generation of key data products, the most relevant being the Elasticsearch vector database providing a hybrid keyword-semantic search service to a battery of ML-products (see Figure 2.1).\n\nKey data products Elasticsearch vector database: Provides hybrid keyword-semantic search service to multiple ML-based products, allowing for efficient retrieval of business-relevant retail items. BigQuery tables: Support custom analysis on items metadata and vector database backfilling.\n\nTakeaways"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 757da07f-aae9-4aae-a3a4-0dab6cc761bf
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Takeaways\n\nThe implemented system provided the necessary data products to fulfill the infrastructure requirements of the client. Appropriate design decisions were key for this result, some worth highlighting.\n\nProcessing items metadata coming from multiple sources"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 26aac92d-2c59-4b36-a47d-77352184ea83
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "An aspect of high importance when designing the data infrastructure for this client was that the metadata of a given item is produced in parallel by multiple teams within the client\u2019s enterprise. Moreover, integration and processing of such metadata needed to be done in near real time. This requirement could be appropriately fulfilled by combining a Kafka server with the Cloud Scheduler and Cloud Run services in GCP. An improvement over the implemented streaming pipeline would be to use Dataflow instead of Cloud Scheduler and Cloud Run, simplifying the design and leveraging a service that is tailored to handle both streaming and batch pipelines."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 507e3067-e3e8-4193-91c4-448211473ccd
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Relevance of hybrid keyword-semantic search in retail\n\nAn advantage of vector databases like Elasticsearch is their ability to combine keyword-based and semantic-based (vector encodings) queries. This is particularly relevant for a retail company, where retrieving items based only on image similarity might not be sufficient to capture the nuances of the items being targeted. Complementing the semantic search with specific text keywords can help cope with such limitations.\n\nServing multiple ML-based products"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 66f38b0c-48f6-49f4-b672-b829e6f5d3fa
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Serving multiple ML-based products\n\nThe core use of the semantic search service was the support of queries from multiple ML-based products achieving specific throughput and latency thresholds. This requirement was successfully met thanks to the scalability features and efficient approximate kNN search algorithm offered by the Elasticsearch vector database. Such a data infrastructure provided the foundations for key ML services, including recommendations and automatic attribute tagging systems, working in concert to increase business productivity.\n\nLooking ahead"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 129afeb5-2102-4501-a9e7-b714cae9ebef
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Looking ahead\n\nIt\u2019s clear that the journey of integrating advanced Data Engineering with AI is both a challenging and immensely rewarding endeavor. The projects we've shared exemplify not just the technical and strategic depths we navigate at Tryolabs but also our commitment to pushing the boundaries of what's possible in AI and analytics.\n\nThe evolution of the AI industry demands continuous innovation and adaptability. Our experiences underscore the importance of solid Data Engineering foundations to not only meet current needs but also to anticipate future challenges and opportunities."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: a84a859a-ef84-48bc-ab3c-31142608d852
encoding: FLOAT32
metadata:
{"title": "Robust Data Engineering: The force propelling AI forward", "link": "https://tryolabs.com/blog/robust-data-engineering-propelling-ai-forward", "language": "en", "description": "Uncover the pivotal role of Data Engineering in advancing AI. This piece walks you through the critical strategies and real-world examples that highlight Data Engineering as the foundational force behind cutting-edge AI developments, driving forward a new era in technology.", "chroma:document": "Stay tuned for for more insights and stories from the forefront of AI and Data Engineering, which will serve as useful references to your path to building AI-based innovative technologies."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: c2508cbc-855d-4ca2-b976-94b3917c6885
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Today marks a significant milestone for us at Tryolabs as we introduce Temporian.\n\nIn collaboration with Google, we've designed this tool to address the multifaceted challenges of temporal data processing head-on. Let's explore the inspiration, functionality, and future of Temporian."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 0c740120-6538-4b46-ada2-46d5933c1f9a
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "In the world of data science, our experience working with clients has made it abundantly clear that temporal data is everywhere, powering a wide range of use cases and applications. From sales forecasting, fraud detection, and marketing optimization to processing signals from electronic sensors, the challenges presented by temporal data are very diverse. Whether dealing with long time sequences spanning years or precise millisecond measurements, efficiently handling temporal events is an essential asset."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 5411f6e3-8c31-4a70-8091-d211d80bebd9
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Typically, generic data science tools are employed to tackle temporal data manipulation. While many of these tools are versatile enough and can be adapted to perform mostly any data transformation, the number of steps and intricacies required to handle some particular cases often leads to large codebases, especially as projects grow in complexity. This quickly becomes a challenge, often leading to finicky data manipulations, code subtleties, hidden behavior, and bugs. On top of that, inefficient computation becomes commonplace when using tools and operations that were not designed for their original purpose."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 60a0fbbf-71b4-4d2a-ae17-897432f19ee3
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Recognizing the need for a specialized solution for this domain, we partnered with Google to develop Temporian: a world-class, open-source Python library for preprocessing and feature engineering of temporal data."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 1f0c559d-53bd-4af5-b429-5123d9cd2334
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "At its heart lies a clever use of proper data structures and formats, specially designed to handle temporal data in all its forms. By representing data as a generic collection of events with corresponding timestamps\u2014potentially originating from multiple sources, having multiple attributes, and being non-uniformly sampled\u2014Temporian seamlessly adapts to a wide array of temporal data formats. Furthermore, it leverages highly optimized C++ operations underneath, and we have an active development lane to leverage Apache Beam's capabilities for running operations on large-scale data."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 20a34188-03eb-40e6-9df3-63533172887a
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Temporian being purpose-built for temporal data manipulation brings numerous benefits, including streamlined temporal data operations and code that is clear, intuitive, and easy to maintain. As a result, it minimizes the likelihood of bugs or hidden behavior, also providing substantial computational efficiency gains."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 588e8049-0b04-4c35-8d13-36b211e693ee
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Not less important, it integrates with the most popular data science and Machine Learning tools. Its user-centered documentation provides comprehensive tutorials, enabling effortless initiation with Pandas and TensorFlow. From exploratory data analysis in interactive notebooks to deploying robust high-volume processing pipelines, Temporian is geared to facilitate the entire spectrum.\n\nThe story behind our collaboration journey"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 761dd394-b56f-4927-b0fe-826c6eba5579
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "How did the paths of Tryolabs and Google converge in the vast landscape of Machine Learning and data science? The answer lies in a shared aspiration and a mutual recognition of each other's strengths. At Tryolabs, forecasting is one of our main areas of expertise\u2014having applied it to solve real-world use cases that range from Predictive Maintenance of Oil Pipes to Pricing Optimization - and as such, we know first-hand that a purpose-made temporal data preprocessing tool would bring tons of value to the table, especially when it comes to ease of development and time to market of such solutions. Add to that Google\u2019s need for a tool capable of processing zettabytes of data from millions of"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6af76642-5dc1-4b2c-817d-f4e7fcb67dc3
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "of development and time to market of such solutions. Add to that Google\u2019s need for a tool capable of processing zettabytes of data from millions of events worldwide in real-time, and you get both a perfect goal to strive for, and two engineering teams with more than enough expertise to take a swing at it!"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 584787f7-3083-4fe1-9078-1f35d92893c9
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "On top of that, we are no strangers to open-source development, with our home-grown Norfair being one very successful example. Neither is it the first time that we collaborate with Google, since that time we streamed together a live-chat event with some of the Swift for TensorFlow\u2019s core developers, sharing our experience as early users of the library. They were incredibly humble and awesome, and we enjoyed the experience so much that we came back for more."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 2e94ad25-8f63-4903-9c57-a2d626a72544
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Temporian\u2019s journey began almost a year ago, when the teams got together to brainstorm what would be the most valuable features in the solution we aimed to build. From then we underwent a large phase of design, followed by the development of an MVP which was then validated via UX testing with real users. After having made adjustments based on the received feedback, and polishing up and publishing the documentation for the library, we finally decided to publish it for its first-ever users to have a go with it. This is only the start - there\u2019s a long road ahead for Temporian, and we\u2019ll be there for it!"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: e576a190-5e46-42c6-a2e7-4291b644003c
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "The library\u2019s development was not without its challenges\u2014needing, for example, to strike a precise balance in exposing a public API that was simple and intuitive for new users but flexible and powerful enough for advanced ones, to support several execution backends for each available operation, and to quickly become proficient in tools that we\u2019d never crossed paths with. Although challenging, overcoming those and many other issues was little but a learning instance that we gladly embraced\u2014and a breeze to go through when surrounded by teammates with the collective experience of our own.\n\nInside Temporian: a deep dive into its capabilities"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: ab5f0b6d-7616-4d35-9bc2-6a5211f1c119
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Inside Temporian: a deep dive into its capabilities\n\nSo: what exactly does Temporian do? And more importantly, what sets it apart from the myriad of other widely adopted and maintained data preprocessing libraries?\n\nA very simple example of a Temporian script. The snippet calculates the weekly sales for each store, visualizes the output with a plot, and exports the data to a CSV file.\n\nUnlike most data preprocessing libraries, Temporian was designed and built with temporal data\u2014and temporal data only\u2014in mind."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 10e37723-0c29-4e4c-97ec-83efa38d0d58
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Unlike most data preprocessing libraries, Temporian was designed and built with temporal data\u2014and temporal data only\u2014in mind.\n\nThat is, Temporian was built to handle the timestamps associated with each of your data points natively, and that enables it to run certain operations on them much, much more efficiently, both in terms of time and memory - in some cases reaching x1000 speedups compared to the same preprocessing written directly in NumPy! Note that this is by no means because of NumPy being slow or inefficient\u2014it just wasn\u2019t built for temporal data (and, as a matter of fact, Temporian heavily relies on it under the hood!)."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: bdd23796-9263-486e-abdb-cab4b4ad7046
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "These results are only made possible by Temporian\u2019s custom data structure, called the EventSet , which arranges temporal data in a way that makes it efficient to operate on it, and to great portions of Temporian\u2019s operations being implemented in C++ and highly optimized.\n\nTemporian\u2019s EventSet isn\u2019t just able to represent time series (which can be defined as uniformly-sampled temporal data, or a series of values that happen sequentially and with equal intervals between them), but also multivariate and hierarchically-structured data, and data with non-uniform and non-aligned sampling, which by itself sets it apart from any alternative libraries that we know of."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6c1d87f5-d857-4eb7-b87a-f423347a013d
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "You may be wondering exactly what operations Temporian provides. The library\u2019s API Reference hosts the complete list of available operators (which is growing by the day!), but in short, Temporian offers a battery of time-related functions, that allow users to compute features as simple as the day of the month an event occurred on, to as complex as the moving standard deviation of the sales of each product in your online store, in relation to the lagged cumulative sum of sales across all products since launch."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 85ce191c-2e35-4047-8dba-f02f2e371266
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "To top it all off: a Temporian program can be developed iteratively in an interactive development environment, such as a Jupyter notebook, and then saved to a file to be run in Apache Beam on production-grade workflows! This is only possible thanks to Temporian tracking the operations a user performs on their data and maintaining an underlying graph of operations - all in a transparent way to most users!\n\nReal-world scenarios\n\nTemporian can be helpful in any domain where temporality plays a role\u2026 Which is just about all of them?"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: c33701a5-323a-40a1-bed9-ba905f9b426b
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Real-world scenarios\n\nTemporian can be helpful in any domain where temporality plays a role\u2026 Which is just about all of them?\n\nJokes aside, temporal data is as commonplace in AI as you get, and can come up in the form of transactions, user events, logs, product sales, medical data, sensor signals, or weather patterns - just to name a few. And in each of those use cases, Temporian can lend a huge hand when it comes to consuming the raw temporal data and outputting something a Machine Learning model can work with!"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 049eeed7-3a57-4916-95fa-4772770fd4e0
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Not sold yet? Check out how Temporian deals with a real-world scenario in the M5 Competition tutorial, which uses Temporian to preprocess and feature engineer hierarchical sales data from Walmart stores across the US. Its output is then fed to a Gradient Boosted Trees model to reach near state-of-the-art performance on forecasting future sales!\n\nThis tutorial gives a good overview of Temporian\u2019s versatility. The competition\u2019s data is loaded into three Temporian EventSet s, which are used to compute a variety of temporal features:\n\nPast sales : the lagged sales values from the last couple of days, computed using lag() ."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: e22d3c66-3879-456c-a39b-957af70710c2
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Past sales : the lagged sales values from the last couple of days, computed using lag() .\n\n: the lagged sales values from the last couple of days, computed using . Moving statistics : the average and standard deviation of sales in the last couple of weeks, computed using simple_moving_average() and moving_standard_deviation() .\n\n: the average and standard deviation of sales in the last couple of weeks, computed using and . Calendar data : the day of the week, day of the month, and month associated with each date we want to forecast on, computed using calendar_day_of_week() , calendar_day_of_month() , and calendar_month() ."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: a7d12e34-eebf-4469-847e-8109a35b3921
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": ": the day of the week, day of the month, and month associated with each date we want to forecast on, computed using , , and . Sales aggregated per department : the sum of sales in each product\u2019s department during the last month, computed using drop_index() , moving_sum() , and propagate() .\n\n: the sum of sales in each product\u2019s department during the last month, computed using , , and . Special events: the number of days remaining until key calendar events, such as Christmas or Father\u2019s Day, computed using propagate() , leak() , and resample() ."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 00c5ddef-f19b-432c-826d-6f484f517c24
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Note that Temporian not only makes it easy for the developer to write the code to compute these features but also executes it extremely efficiently, delegating as much processing as possible to C++ and minimizing memory usage, which can make or break your program when dealing with large amounts of data!\n\nIf you want other examples of Temporian at work (or just can\u2019t wait to dive right into the code), check out the Getting Started guide for a simple step-by-step example, or one of the featured tutorials for Detecting payment card fraud, Heart Rate Analysis, and Loan Outcomes Prediction!\n\nKickstarting your Temporian experience"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 321a096b-75a2-4fda-97da-e10735b08a1a
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Kickstarting your Temporian experience\n\nThe best place to start your journey as a Temporian user is the library\u2019s docs, which provide all kinds of resources for new and advanced users, including a Getting Started tutorial, a quick intro to how the library works, a full tour of its concepts and capabilities, and the complete API Reference for the library\u2019s components.\n\nIf instead you\u2019re more of a hands-on learner and can\u2019t wait to get started: Temporian is available on PyPI and can be installed with a simple pip install temporian (though we do recommend checking out some of the introductory guides first!).\n\nEmbracing open-source, community, and the future"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 8da6b1a6-f45c-4aac-9e6f-29b83b7b168e
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Embracing open-source, community, and the future\n\nThis exciting collaboration between Tryolabs and Google holds greater significance since we're creating an open-source library, accessible for use and improvement by anyone worldwide. We know for a fact that the best solutions emerge when the collective intelligence of the community comes together, and thus, we welcome experts and developers alike to give it a try and join forces with us!"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 2991db2a-c5e2-434e-a3a4-3a4e0365be25
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Now is the perfect time to get involved as a user, developer, or early adopter, helping us showcase the library in real-world scenarios. With an increasingly stable API and comprehensive, well-tested documentation, at this point you can confidently explore and leverage Temporian's capabilities. Moreover, user feedback and feature requests will play an important role in shaping the library's future, ensuring that it continuously evolves to meet the needs of data scientists, engineers, researchers, and all potential users."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: e63eb193-c12a-43ee-b9de-9fbb1cd36063
encoding: FLOAT32
metadata:
{"title": "Introducing Temporian: Tryolabs and Google venture in temporal data processing", "link": "https://tryolabs.com/blog/tryolabs-google-introducing-temporian", "language": "en", "description": "Temporian is an open-source Python library for efficient temporal data processing. Developed in collaboration with Google, it addresses the challenges of working with time series data. Learn how its specialized design minimizes bugs and boosts productivity.", "chroma:document": "Looking ahead, our roadmap is ambitious and promising. We will continue to develop Temporian, expanding its capabilities with more operators, enhancing Apache Beam and cloud providers integration, and supporting real-time streaming applications. Staying focused on temporal data preprocessing and feature engineering, we aim to become the go-to library for industry projects at all scales.\n\nWe are thrilled to be on this journey, creating a top-notch tool that bridges a crucial gap in the data science landscape. We invite you all to join us in making Temporian the perfect solution for temporal data processing!"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 9f20bf5a-10f0-4e9a-99bf-e505f005ea95
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "In the ever-evolving landscape of natural language processing, the advent of Large Language Models (LLMs) has ushered in a new era of possibilities and challenges. While these models showcase remarkable capabilities in generating human-like text, the potential for unintended consequences raises concerns: hallucinations\u2014instances where the model generates information that is factually incorrect or entirely fabricated, formatting problems, the use of harmful and biased language, and security issues to name a few. Generative AI is a double-edged sword and what makes LLMs helpful might also introduce significant risks."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d3ed82f2-777f-46b0-b6da-ea59e169b30c
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "To mitigate some of these issues, researchers are working on what they call model alignment, using strategies such as Reinforcement Learning from Human Feedback (RLHF) and fine-tuning. However, those strategies are often implemented before making the LLM available to the public. In this post, we will delve into techniques that can be used even when the model has been developed. For a more in depth exploration of these techniques, refer to this survey."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6a265110-17fe-4cae-a645-8b5571ff9b1f
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Understanding the need to address these challenges, developers often seek alternative ways to limit the outputs of LLMs to ensure the practical usability of LLM-generated content in diverse applications, enhance accuracy and security, and maintain ethical standards. As we explore strategies for controlling LLM outputs, we aim to strike a balance between harnessing the power of these models and addressing the inherent challenges they pose.\n\nWe\u2019ve researched alternatives to address output limitation so you don\u2019t have to. At a high level, there are 3 strategies that act at different stages of the process to \u2018control\u2019 an LLM:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 1a144045-92bd-4459-9d8f-e06548dcec35
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Effective prompting Parameters and guided generation Post-generation guardrails\n\nStrategy 1: effective prompting\n\nPrompting is the easiest strategy we\u2019ll cover. Prompting, in essence, is about designing the initial query or statement in such a way that it effectively communicates our intent to the model, thereby influencing its response to align with our desired outcome. As the field advances, its focus expands beyond crafting prompts to encompass the design of systems where model queries serve as integral components."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: c97ca543-e32e-4b3c-9332-a0df10b2f21d
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Utilizing prompts represents a potent approach to augment the adaptability and manageability of large language models. A meticulously crafted prompt holds the potential to significantly amplify the productivity of LLMs. Some strategies that use prompting are templates, example libraries, or prompt optimization tools that structure user-inputted prompts. The advantages of these strategies are facilitating effortless interaction for non-technical users as well as optimizing interaction efficiency."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: ab6c26ac-cebe-4e00-a21f-5fabbc81bfdf
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Chain of Thought (CoT) is one example of how a prompt can be adapted to align with an expected result. It\u2019s a method that encourages the LLM to justify its \u201creasoning\u201d and usually involves including examples on the prompt that better illustrate the intended answer.\n\nAwesome ChatGPT Prompts is an open-source collection of curated prompt templates for ChatGPT, each tailored for unique applications and use cases. LangGPT follows a similar approach. As an example, here\u2019s a prompt template to turn ChatGPT into a Linux Terminal:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 97b7f700-e344-499a-b758-ecf1e08629c6
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "PromptSource is a toolkit for creating, sharing, and using natural language prompts. Prompts are saved in standalone structured files and are written in a simple templating language called Jinja. An example of a prompt available in PromptSource for SNLI is:\n\nPromptLang is a custom programming language specifically designed for use in GPT-4 prompts and AI interactions. Promptify is used to solve Natural Language Processing (NLP) problems by generating different NLP task prompts for popular generative models like GPT, PaLM, and more.\n\nStrategy 2: parameters and guided generation"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: dfd6dcd8-6dad-4277-b6c7-88563843d3d5
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Moving on to the second category of strategies, we focus on the techniques employed when calling the LLM. This stage involves the use of specific methods that can further refine the model's output during the generation process. One such powerful tool is logit filtering. Before covering this strategy, let\u2019s discuss how text generation works under the hood. After getting the prompt, the model generates logits (raw, unnormalized predictions) for each possible output token (which isn\u2019t necessarily a word). These logits are then passed to a softmax function, generating a probability for each possible output and a probability distribution over the entire model\u2019s vocabulary."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d42e2e56-1a21-48e5-b2c2-3e86cfadf9a0
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "By accessing the logits themselves, we can manipulate them and scope the response, for example by:\n\nSetting some logits to zero: useful to filter out words, topics, etc.\n\nApplying a weighted mask: useful to allow for more control and aligns with a probabilistic approach.\n\nIncreasing/decreasing desired logits: useful to adjust the chances of some tokens being generated over others.\n\nTake into account that directly manipulating logits can sometimes lead to unnatural responses if done too aggressively"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: c9703e0d-4db3-4663-8feb-78c3a832305f
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Take into account that directly manipulating logits can sometimes lead to unnatural responses if done too aggressively\n\nLM-Format-Enforcer is a library that enforces the output format of language models by filtering tokens. It works by combining a character-level parser and a tokenizer prefix tree into a smart token filtering mechanism. When the language model generates a token, the character level parser gets advanced according to the new characters, ready to filter the next timestep.\n\nHere is an example using the regular expression format Michael Jordan was Born in (\\d)+."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: f054446b-196b-424c-8bdd-d7d380da00e2
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Here is an example using the regular expression format Michael Jordan was Born in (\\d)+.\n\nidx generated_token leading_token generated_score leading_score generated_token_idx leading_token_idx 0 \u2581 \u2581 1.000000 1.000000 29871 29871 1 Michael \u2581Sure 0.000027 0.959473 24083 18585 2 \u2581Jordan \u2581Jordan 1.000000 1.000000 18284 18284 3 \u2581was \u2581was 1.000000 1.000000 471 471 4 \u2581Born \u2581born 0.000008 1.000000 19298 6345 5 \u2581in \u2581in 0.994629 0.994629 297 297 6 \u2581 \u2581 0.982422 0.982422 29871 29871 7 1 1 1.000000 1.000000 29896 29896 8 9 9 1.000000 1.000000 29929 29929 9 6 6 1.000000 1.000000 29953 29953 10 3 3 1.000000 1.000000 29941 29941 11 . . 0.999512 0.999512 29889 29889 12 < /s > < /s > 0.981445 0.981445 2 2"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 88ccd760-2a32-4b06-9f47-c613a34bc5ba
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "The model \"wanted\" to start the answer using Sure , but the format enforcer forced it to use Michael . Afterward, almost all of the leading scores are within the allowed token set, meaning the model likely did not hallucinate due to the token forcing. The only exception was timestep 4 - \" Born\" was forced while the LLM wanted to choose \"born\". This is a hint for the prompt engineer, to change the prompt to use a lowercase \u201cb\u201d instead."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 52c500dc-46bd-4cb4-9bbd-32ef02f5bc43
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "The obvious limitation of this method is that it requires a Python API to process the output logits of the LLM. This means that, at least for now, it cannot be used with OpenAI ChatGPT and similar API-based solutions because those APIs don\u2019t offer that kind of access."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: c9ce1886-5cd9-415f-81c8-1ab51646d13d
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "ReLLM (an acronym for \u2018Regular Expressions for Language Model Completions\u2019) is another library that works at the token level. After providing a prompt and a regular expression, ReLLM filters non-matching tokens pre-generation. For each token, ReLLM tests every possible completion against a partial regex. For the potential completions that do not match the pattern, ReLLM masks the logits so that the language model does not generate them. Here is an example of GPT2 responses with and without ReLLM:\n\nParserLLM extends ReLLM to handle context-free grammar and a parser generator to determine valid next tokens for an LLM generation."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 9ff5090a-ab6e-4e7e-b551-2cd08922ccf4
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "ParserLLM extends ReLLM to handle context-free grammar and a parser generator to determine valid next tokens for an LLM generation.\n\nAnother key strategy when calling the LLM is tuning certain parameters and leveraging the features exposed by the LLM. It is worth taking the time to familiarize yourself with the documentation and parameters of the model you intend to use."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 611ae7c9-2a06-463b-9819-15a000cb1ef1
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "One example of this is OpenAI\u2019s function calling, which \u201callows you to more reliably get structured data back from the model\u201d. Another strategy is using the JSON mode, although OpenAI\u2019s documentation includes the following disclaimer: \u201cWhile this does work in some cases, occasionally the models may generate output that does not parse to valid JSON objects\u201d.\n\nAnother strategy is using OpenAI\u2019s Moderation endpoint, free to use when monitoring the inputs and outputs of OpenAI APIs, which classifies the following categories: hate, threatening, harassment, self-harm, sexual, minors, and violence. The endpoint returns the following fields:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 069b09c6-c6ba-4fad-a357-3c332f0fa89c
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "flagged : Set to true if the model classifies the content as violating OpenAI's usage policies, false otherwise.\n\n: Set to if the model classifies the content as violating OpenAI's usage policies, otherwise. categories : Contains a dictionary of per-category binary usage policies violation flags. For each category, the value is true if the model flags the corresponding category as violated, false otherwise."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 70b79e65-ae80-4426-b14e-7e69c966fc5b
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": ": Contains a dictionary of per-category binary usage policies violation flags. For each category, the value is if the model flags the corresponding category as violated, otherwise. category_scores : Contains a dictionary of per-category raw scores output by the model, denoting the model's confidence that the input violates the OpenAI's policy for the category. The value is between 0 and 1, where higher values denote higher confidence. The scores should not be interpreted as probabilities."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 91bcbbcb-04fc-4c11-95dd-0cf824938779
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Most LLMs also provide parameters that can be tuned. Parameters such as temperature , top-k and top-p sampling can be used to somewhat control the model\u2019s output. The temperature strikes a balance between randomness and predictability, adjusting the probability distribution used for token selection. A higher temperature increases randomness by flattening the distribution, while a lower temperature concentrates probability on a smaller set of tokens, resulting in more deterministic output. Top-k just considers the K most likely tokens and ignores all others, then samples one of them by considering their likelihood scores. Top-p dynamically sets the size of the shortlist of tokens whose sum"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d271a1c2-80a3-456c-8be6-285b4e3305ca
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "all others, then samples one of them by considering their likelihood scores. Top-p dynamically sets the size of the shortlist of tokens whose sum of likelihoods does not exceed a certain value p."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 7b07749a-db8d-400a-bdb1-73e354d8b4aa
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Another interesting parameter is known as logit_bias . It can be used to exclude words (or actually, tokens) from the generated response. The way it works is the following: a user can specify a bias (positive or negative) to add/subtract to the logits of a specific token generated by the model before sampling."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 52922e67-4802-42f3-9a62-17d016807bbe
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "As stated in OpenAI's API documentation, the impact of specific values will differ depending on the individual model used. Typically, values ranging between -1 and 1 tend to decrease or increase the likelihood of a token being selected, respectively. In contrast, extreme values such as -100 or 100 are likely to result in either a complete ban or an exclusive selection of the relevant token."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 1b26e21b-c0ec-44b1-9059-aae51e88c3e3
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "More advanced techniques can be used to mitigate certain behaviors. For example, DoLa (Decoding by Contrasting Layers) is a decoding strategy for reducing hallucinations with pretrained LLMs. The approach obtains the next-token distribution by contrasting the differences in logits obtained from projecting the later layers versus earlier layers to the vocabulary space, exploiting the fact that factual knowledge in an LLMs has generally been shown to be localized to particular transformer layers.\n\nBear in mind that a caveat of Strategy 2 is the need to have a more granular access to the LLM (e.g. logits), because APIs don\u2019t tend to expose everything that might be needed."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: bd05c29f-6da7-4f77-9140-7dd4b30ed578
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Strategy 3: post-generation control\n\nPrompts are a flexible way to implement limitations, but they cannot guarantee that the same model will generate the same output for the same prompt.\n\nSecurity guardrails are a way of letting the user add structure, type, and quality guarantees to the outputs of LLMs. Examples of guardrails include semantic validation (checking for bias or harmful content in generated text, checking for bugs in generated code, etc.) and enforcing structure and type guarantees (e.g. JSON)."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 1f12c78e-5aa7-4cf2-b038-1f3b5bb2064c
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "This technique works with the LLMs output, providing a posteriori methods that guarantee that the output will match certain criteria: regular expressions, follow a JSON schema, be parseable into a Pydantic model, etc.\n\nExamples\n\nGuardrails does pydantic-style validation of LLM outputs (including semantic validation), takes corrective actions (e.g. re-asking LLM or filtering out the invalid output) when validation fails, and enforces structure and type guarantees (e.g. JSON). Under the hood, it provides a file format ( .rail ) for enforcing a specification on an LLM output, and a lightweight wrapper around LLM API calls to implement this spec."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: eba86627-b25b-4af0-aaea-c7c41a0cbd8a
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Let\u2019s review some validators from the library to better understand the kind of checks that can be applied. Some are used to check code (e.g. BugFreePython , BugFreeSQL , DetectSecrets , ExcludeSqlPredicates , SqlColumnPresence ). Others have to do with text format (e.g. EndsWith , LowerCase , OneLine , RegexMatch , TwoWords ) and some validate the text\u2019s content itself (e.g. Provenance , IsProfanityFree , CompetitorCheck ). Users can also create their own custom validator that enforces what the application is intended to do.\n\nGuardrails logs all of the steps it takes to enforce the RAIL spec, which can later be inspected. Here\u2019s a diagram with the full flow:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 8c06e0cf-09db-4974-a4ef-43e8ab467ec0
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Guardrails logs all of the steps it takes to enforce the RAIL spec, which can later be inspected. Here\u2019s a diagram with the full flow:\n\nOne validator called OnTopic can be used to verify that a response is related to a provided topic (or one of many provided topics). Under the hood, it uses an ensemble of Zero-Shot classification model combined with an LLM to determine if the condition is met. Here\u2019s an example that illustrates how to use it:\n\nWe recently collaborated with Guardrails in the definition and implementation of some Validators, including OnTopic.\n\nAlternatively, NeMo Guardrails (an open-source toolkit developed by NVIDIA) and Guardrail ML offer similar features."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6dde9adf-62f1-4398-a4f1-759546ad4c24
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Alternatively, NeMo Guardrails (an open-source toolkit developed by NVIDIA) and Guardrail ML offer similar features.\n\nIn addition to guardrails, there are countless strategies when processing the response. One example is SelfCheckGPT, a zero-resource hallucination detection solution that can be applied to black-box systems. The basic idea behind it is that if an LLM knows a given concept, sampled responses are likely to be similar and contain consistent facts. By sampling multiple responses from an LLM, one can measure information consistency between the different responses and determine if statements are factual or hallucinated."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d027ef87-fdab-47fc-bc87-7610cb4ecefc
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Output parsers are a simple way to get more structured information from the LLM\u2019s response or enforce an expected format. LangChain, a framework for developing applications powered by language models, offers an implementation and this example to illustrate how to use it:\n\nThere are other similar libraries (e.g. Guidance, JsonFormer, Outlines, Griptape) that also work by using different techniques to enforce an output format.\n\nConclusions\n\nEach of these techniques has its unique strengths and weaknesses, and their applicability depends on the specific requirements of the task at hand and the kind of output filtering intended."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: a1e9bf49-0ee3-47ed-843a-a0a2c015c505
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Prompt tuning is a powerful tool for guiding the model's output. By carefully crafting the input prompt, one can influence the model's responses. However, this method requires a deep understanding of how the model interprets prompts and may not always yield consistent results due to the inherent randomness in the model's output.\n\nLogit filtering is a more advanced technique that involves manipulating the model's logits to prevent it from generating certain words or phrases. This method can be highly effective, but it requires a deep understanding of the model's inner workings and can be computationally expensive."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 75af9288-68a0-48a0-9afb-fa115d2afe27
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Parameter tuning, such as adjusting the temperature and top-k sampling, provides a way to control the randomness and diversity of the model's output. While these methods offer a degree of control, they do not guarantee the elimination of harmful or inappropriate content.\n\nGuardrails are a proactive approach to limit the model's output by setting predefined rules or boundaries. This method can be effective in preventing the model from generating certain types of content. However, it may also limit the model's creativity and versatility, and there's always a risk that some unwanted content may slip through."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d590861b-4eca-4078-813a-0f92d2961408
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "Parsing involves analyzing the model's output after it has been generated, allowing for the removal or modification of unwanted content. This method can be effective, but it is reactive rather than proactive, meaning that harmful or inappropriate content may be generated before it is caught and removed."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 65bb65d6-69ea-4161-b3ef-b5b629414c4f
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "In conclusion, the choice of strategy for controlling and limiting the output of LLMs depends on the specific needs and constraints of the task at hand. A combination of these techniques may often be the best approach, providing a balance between harnessing the power of LLMs and mitigating their potential risks. Also keep in mind that if you want to limit a particular type of output, there may be more strategies for that specific behavior. For example, this survey gathers many hallucination mitigation techniques."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: a16642c1-8279-474e-a01f-11830f8a82a9
encoding: FLOAT32
metadata:
{"title": "Taming LLMs: strategies and tools for controlling responses", "link": "https://tryolabs.com/blog/strategies-and-tools-for-controlling-responses", "language": "en", "description": "Discover effective strategies for controlling Large Language Models (LLMs). Learn about prompting, logit filtering, parameter tuning, and guardrails to improve LLM outputs.", "chroma:document": "It's important to note that the field of controlling and limiting the output of LLMs is an active area of research. As we continue to explore the capabilities and challenges of these models, new techniques and tools are being developed continuously. This ongoing innovation is expanding our toolkit for managing LLMs, enabling us to better align these models with our goals and values. The strategies discussed in this post represent the current state of the art, but we can expect to see further advancements in the near future. As we navigate this evolving landscape, the key is to remain adaptable, continually learning and applying new methods as they emerge."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 72c72e8d-bc4a-48c9-9d18-f8137485876e
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "As Artificial Intelligence (AI) continues to revolutionize industries and redefine operational strategies, the challenge emerges: how can enterprises efficiently deploy, manage, and scale these AI solutions? Navigating the intricacies of AI development and deployment can be a daunting task. This is where MLOps shines. Merging the best of Machine Learning with operational practices, MLOps is more than just a technical methodology\u2014it's the strategic compass for AI-driven enterprises. In this exploration, we'll delve into the significance of MLOps, highlighting its pivotal role in translating the vast potential of AI into tangible business results. Embark with us on this illuminating journey."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 853ab504-f2af-4085-bcb9-69cd17aaac4f
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Why MLOps is Crucial for AI-Driven Organizations\n\nAI promises transformative power. From automation to predictive analytics, its potential applications are vast and varied. But harnessing that power in a way that aligns with business goals requires more than just stellar algorithms or accurate models. It requires a structured process that integrates AI advancements seamlessly into an organization's workflow. This is where MLOps comes to the fore.\n\nConnecting AI Objectives with Business Ambitions"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 9a227d1f-5e87-45b6-bf2c-aaef058f9819
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Connecting AI Objectives with Business Ambitions\n\nBefore delving deep into the technicalities, it's paramount to understand the bigger picture: How does AI align with the overarching business objectives? Think of AI as the engine and MLOps as the steering wheel. While AI powers the drive, MLOps ensures the journey is in the right direction, aligning with the destination, the organization's business ambitions.\n\nWith MLOps, organizations can:\n\nEstablish clear metrics that tie AI performance to business outcomes.\n\nStreamline the feedback loop, ensuring AI developments continually cater to evolving business needs."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 7bca51d4-0e25-41a7-bc7f-01049791eef6
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Streamline the feedback loop, ensuring AI developments continually cater to evolving business needs.\n\nFoster collaboration between data scientists and business stakeholders, ensuring both are on the same strategic page.\n\nEnsuring reproducibility, efficiency, and speed of AI deployments\n\nMLOps isn't just about direction; it's also about efficiency and momentum. As AI-driven solutions scale, ensuring consistent performance becomes a challenge. Variability in data, environmental conditions, and deployment platforms can introduce unintended consequences or inefficiencies.\n\nKey benefits of MLOps in this arena include:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 0852ef82-477a-42b2-8833-6f523002d76f
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Key benefits of MLOps in this arena include:\n\nReproducibility : With standardized processes and environments, models can be deployed and re-deployed consistently, ensuring stable performance over time.\n\n: With standardized processes and environments, models can be deployed and re-deployed consistently, ensuring stable performance over time. Efficiency : MLOps introduces practices like automated testing and continuous integration, minimizing manual interventions and reducing the chances of errors."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 7dc3ed88-1152-4406-a05f-4169d3f375e0
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": ": MLOps introduces practices like automated testing and continuous integration, minimizing manual interventions and reducing the chances of errors. Speed: Through streamlined pipelines, AI solutions can be swiftly moved from development to deployment, ensuring businesses can rapidly capitalize on AI insights and advantages."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 920920bd-2699-4c92-8a2c-acc3b1b414cc
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "In essence, MLOps serves as the bridge between the dynamism of AI and the structured ambitions of business enterprises. It's the linchpin ensuring AI doesn't just operate in a vacuum but is intricately and strategically tied to the broader organizational tapestry. By instilling reproducibility, efficiency, and speed into AI deployments, MLOps ensures that AI-driven organizations are not just running, but running in the right direction, at the right pace.\n\nThe MLOps Journey: Key Stages Explained\n\nStage 1: Getting your Data in order"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 3b70a17f-c859-4f21-96c0-c145abb24e95
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "The MLOps Journey: Key Stages Explained\n\nStage 1: Getting your Data in order\n\nThe first step of any MLOps journey starts at the heart of every Machine Learning project: the data. Quality data is the foundation of successful AI, and initial efforts to enhance its usability can drastically boost the efficacy of your Data Science teams.\n\nCommon data challenges: Lack of centralization: Data dispersed across numerous databases creates a barrier to efficient data integration and interpretation."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 7e8720b0-a91c-4f8b-8b9a-0222c09b830c
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Data dispersed across numerous databases creates a barrier to efficient data integration and interpretation. Poor data quality: Unreliable datasets plagued with missing values, free-text categorical descriptors, and other basic issues compromise the validity of results.\n\nUnreliable datasets plagued with missing values, free-text categorical descriptors, and other basic issues compromise the validity of results. Lack of documentation: Without a centralized repository detailing data specifics, teams over-rely on the tacit knowledge of seasoned members to navigate the data maze."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: bd1cf693-752a-40d7-b245-d83d4d014c8a
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Without a centralized repository detailing data specifics, teams over-rely on the tacit knowledge of seasoned members to navigate the data maze. No reusability: If each model requires unique SQL queries and lacks standard preprocessing, the wheel is being reinvented every time \u2013 an inefficient and error-prone process.\n\nStrategies for Data enhancement:\n\nCentralization: Use a Data Warehouse like BigQuery or Snowflake to consolidate data. When this isn't viable, Data Catalogs like Alation, Amundsen, or AWS Glue can provide an organized directory of information across services."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 4b18a5b3-6525-4a8c-89ff-0e318b965b95
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Use a Data Warehouse like BigQuery or Snowflake to consolidate data. When this isn't viable, Data Catalogs like Alation, Amundsen, or AWS Glue can provide an organized directory of information across services. Data quality improvement: Regular manual checks by Data Scientists can flag potential issues. Open communication channels between data creators (typically Engineering and Data Engineering teams) and consumers (like Data Science teams) allow for collaborative issue resolution."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 067919ea-e976-4f6d-8c9a-8d21718c372c
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Regular manual checks by Data Scientists can flag potential issues. Open communication channels between data creators (typically Engineering and Data Engineering teams) and consumers (like Data Science teams) allow for collaborative issue resolution. Prioritize documentation: Data documentation is as pivotal as code documentation. Templates like Datasheets for Datasets, the Aether Data Documentation Template, or Data Nutrition Labels offer frameworks to record essential data characteristics and provenance."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 02cf462b-a0a4-4fc3-8c86-8006a1d3845a
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Data documentation is as pivotal as code documentation. Templates like Datasheets for Datasets, the Aether Data Documentation Template, or Data Nutrition Labels offer frameworks to record essential data characteristics and provenance. Promote reusability: Encourage shared, stored queries. While a simple .sql file in a repository might suffice, creating views in your Data Warehouse can be even more beneficial. Regularly allocate time for team members to generalize and share these queries. Aim to shift data cleaning to the ETL phase, providing tables with pre-cleaned data."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 5a849f4a-bfd6-48dc-9f2c-02d51707a8ab
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Although vital, this blogpost doesn't delve into data security, compliance, and user privacy. However, mastering these foundational data steps allows for progressing to more intricate facets of your MLOps infrastructure.\n\nStage 2: Experiment and model tracking\n\nAs data teams progress from ensuring data quality and usability, the next pivotal stage in the MLOps journey is experiment and model tracking. This stage revolves around the systematic documentation, monitoring, and management of experiments and models. But why is this process so essential, and what challenges might one face?\n\nChallenges in experimentation:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: b825253a-5bd7-4ab9-be2e-be4de20ad6ae
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Difficulty reproducing results: Without structured tracking, re-creating experiments to achieve the same results becomes a maze. Losing track can translate to lost time, effort, and critical insights, acting as significant hindrances. Collaboration barriers: The absence of a centralized system can lead to disjointed team efforts. Imagine multiple data scientists working independently on their Jupyter notebooks without versioning, causing a significant overlap and rework. Inefficient experimentation: Tracking the myriad of experiments, understanding their outcomes, and discerning the reasons for their success or failure can get chaotic. An unorganized approach can slow down the"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 7694eaba-402a-4af9-aca9-789cd1250a24
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "understanding their outcomes, and discerning the reasons for their success or failure can get chaotic. An unorganized approach can slow down the experimentation process considerably. Inability to track model performance: Choosing the best model becomes an uphill task when there\u2019s no systematic way to compare the performance metrics of different models."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: b5f327f6-90a8-416b-9863-e6228e645a6b
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "As organizations scale, these challenges can snowball, leading to considerable bottlenecks and stymied development. However, addressing these issues head-on can smooth out the workflow significantly.\n\nSolutions and approaches\n\nStandardized experimentation environment: The foundation lies in ensuring that the entire Data Science team operates in a standardized setting. Tools like pyenv coupled with poetry or pip-tools can aid in managing Python dependencies. Moreover, integrating Docker into the workflow helps create containerized environments, making them shareable and consistent across various stages. Environment management with Pyenv"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: cb335097-b0fc-431a-9c57-0d8f2a1291c4
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Embrace experiment tracking tools: Tools dedicated to tracking experiments can be game-changers. They meticulously document everything from hyperparameters and input data to performance metrics, enabling data scientists to draw comparisons, spot trends, and assimilate learnings from past experiments. Examples of experiment-tracking tools include Weights & Biases, Comet ML, Studio by Iterative.ai, and even TensorBoard. You may also track your experiments directly on GCP Vertex AI or AWS SageMaker Experiments features."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: b8d15397-97ee-461a-9868-441497f9a07c
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Tools dedicated to tracking experiments can be game-changers. They meticulously document everything from hyperparameters and input data to performance metrics, enabling data scientists to draw comparisons, spot trends, and assimilate learnings from past experiments. Examples of experiment-tracking tools include Weights & Biases, Comet ML, Studio by Iterative.ai, and even TensorBoard. You may also track your experiments directly on GCP Vertex AI or AWS SageMaker Experiments features. Adopting a model registry: Think of a Model Registry as a centralized library for your trained models. Such a registry helps streamline model lifecycle management, making collaboration a breeze and ensuring that"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 51af1dd1-f274-449d-9754-b048db2c6834
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "library for your trained models. Such a registry helps streamline model lifecycle management, making collaboration a breeze and ensuring that insights and models are retained and not lost in the shuffle. While GCP Vertex AI and AWS Sagemaker offer built-in registry capabilities, they often require a custom interaction layer to suit specific organizational needs."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: af4889a5-f721-4afa-a1d3-5a0c053a8b03
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "In this stage, the importance of structured and systematic tracking of experiments and models cannot be understated. Adopting the right tools and practices ensures that the organization can scale its Machine Learning efforts without getting entangled in its own web of experiments.\n\nStage 3: Standardizing production process\n\nUpon streamlining data quality and ensuring a well-documented trail of experiments and models, teams often hit a new roadblock: transitioning models from proof of concept to production swiftly and efficiently.\n\nProduction challenges"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: abccf960-a9cb-4edc-8c81-deeb15e9762c
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Production challenges\n\nLack of reusable code: Every model's deployment seems like reinventing the wheel, with custom code and infrastructure required every single time.\n\nEvery model's deployment seems like reinventing the wheel, with custom code and infrastructure required every single time. Maintainability concerns: Ensuring active models function flawlessly in production becomes a challenge with the increasing complexity of code and infrastructure."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: af54221a-6e8d-4a5d-b7cb-970865187676
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Ensuring active models function flawlessly in production becomes a challenge with the increasing complexity of code and infrastructure. Delay in deployment: The gap between acknowledging a model's efficacy and its first live prediction in production feels extended and cumbersome.\n\nTo swiftly navigate these challenges, the key lies in consistency in development environments and strategic model-serving approaches.\n\nModel serving approaches\n\nThere are two predominant methods for serving Machine Learning models. These methods, each with its own unique approach, are integral to the process of making predictions and interpreting results. Let's take a deeper look:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d83fb83a-c380-412b-a639-728bb2317e66
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Batch Predictions: This method involves a series of systematic steps. Initially, scripts or jobs pull data, which is then preprocessed. The model is loaded, predictions are made, and these predictions are diligently stored for future reference."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: f0cbe8e0-db28-4baa-8682-efded5e7daaf
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "This method involves a series of systematic steps. Initially, scripts or jobs pull data, which is then preprocessed. The model is loaded, predictions are made, and these predictions are diligently stored for future reference. Online Prediction APIs: This process is a bit more immediate. The model's raw features are inputted, undergoing only light preprocessing. Predictions are swiftly made, and the results are immediately presented, providing real-time insights."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 5eddd48c-8c80-4ba6-a908-2f36c67a0c59
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Guidelines for Model Serving Instead of reinventing the wheel each time, standard practices streamline the decision-making process concerning model formats, infrastructure, and serving frameworks. Platforms like SageMaker Models (AWS), VertexAI endpoints (GCP), MLFlow Models (Databricks), or NVIDIA Triton offer an almost plug-and-play experience, simplifying model deployment to a few clicks or commands."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 9bacbfa1-324b-45fc-adf4-1abf191d9c64
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Standard Production Environments One may wonder: how can we align AI models with the plethora of deployment options? The answer resides in the standardization of production environments, a technique that, when correctly mirrored to development environments, can result in notable time and effort savings. The use of dependency managers (examples include poetry and pip-tools) adequately caters to a majority of development needs. However, when transitioning into production, a more robust solution is often required for consistency and predictability. In this respect, Docker stands out as a reliable choice, effectively minimizing unwelcome surprises during deployment."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 51b9c0ea-0458-484d-be30-6607a4e1d713
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "The same way you would use a dependency manager to standardize a development environment, we recommend using Docker for a standard production environment.\n\nMyriad strategies exist to standardize and expedite the transition from PoC to production. Identifying the perfect blend that complements the team's capabilities and the tech stack is pivotal. Prioritizing consistency in both development and production environments ensures that models trained are seamlessly integrated into the standardized serving process.\n\nStage 4: Model and Data Monitoring"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: ddafa8fa-bfef-4537-ae7b-9e5af1ec75e3
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Stage 4: Model and Data Monitoring\n\nAs organizations standardize the serving process of models, the logical next step is scalability. Initially, manual oversight might suffice for monitoring performance, but true scalability demands automation in model and data monitoring. Without this automated vigilance, a model's efficiency can plummet, adversely affecting end-users, oftentimes unbeknownst to the organization until complaints start pouring in.\n\nRecognizing the need for monitoring"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: a7cb72cf-91c0-4b35-9e4f-f10f00ff6769
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Recognizing the need for monitoring\n\nExternal feedback: Are most of your insights on model performance stemming from external feedback? If so, it\u2019s a red flag. Efficient systems should internally detect and rectify issues before end-users even notice.\n\nAre most of your insights on model performance stemming from external feedback? If so, it\u2019s a red flag. Efficient systems should internally detect and rectify issues before end-users even notice. Emerging biases: If the model's outputs display a consistent bias towards specific segments, it's indicative of data bias, which can be a subtle, yet detrimental issue."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 06c9b143-c7cd-456f-aaa9-417ba2d7f048
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "If the model's outputs display a consistent bias towards specific segments, it's indicative of data bias, which can be a subtle, yet detrimental issue. System failures: Just as in traditional software, an ML system's failure should be promptly detected and rectified. Efficient monitoring can be a savior here.\n\nSolutions to consider\n\nData monitoring: Data is the lifeblood of models. Monitoring it can preemptively flag issues like data drift, biases, or missing values. It ensures the model's training data remains representative, maintaining the integrity of its outputs. Moreover, detecting shifts in data distribution can hint towards the need for model retraining."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 8a246fc8-ea13-40ca-a6c5-cca9daf99c74
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Data is the lifeblood of models. Monitoring it can preemptively flag issues like data drift, biases, or missing values. It ensures the model's training data remains representative, maintaining the integrity of its outputs. Moreover, detecting shifts in data distribution can hint towards the need for model retraining. Model monitoring: This is a continuous process of overseeing key performance metrics, often set in collaboration with stakeholders. Early identification of performance dips or biases allows timely interventions \u2013 from retraining to fine-tuning. Furthermore, resource utilization insights can lead to cost optimizations."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 3c197f21-533b-4c62-99f8-b4ec4b5fa7e2
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "This is a continuous process of overseeing key performance metrics, often set in collaboration with stakeholders. Early identification of performance dips or biases allows timely interventions \u2013 from retraining to fine-tuning. Furthermore, resource utilization insights can lead to cost optimizations. Observability: Beyond monitoring, observability aids in understanding the \"why\" behind any issues, facilitating quicker resolutions. This includes deep dives into logs, metrics, and system traces."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: ca896d4f-6f67-47c1-8622-afadab70cd33
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Generic cloud monitoring and observability tools, such as Datadog, AWS CloudWatch, GCP Cloud Operations, or Azure Monitor, provide broad oversight. Sentry and Bugsnag excel in issue detection and troubleshooting. For specialized ML monitoring, Whylabs stands out, offering in-depth insights for data and ML systems. The beauty lies in the harmonious coexistence of these tools. Instead of choosing one over the other, they can be synergistically integrated to craft a comprehensive monitoring and observability solution.\n\nTake the next step in your MLOps journey"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 004f56ad-b784-409d-8d37-ba6fb914b94e
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Take the next step in your MLOps journey\n\nWe've only scratched the surface of what's possible in the Data Science ecosystem. Beyond these preliminary stages, a myriad of strategies, nuances, and insights await you, insights that could revolutionize your organization's approach to AI and data.\n\nAt Tryolabs, we pride ourselves on being more than mere service providers. We are your partners and your strategic allies on the road to MLOps excellence. Our team, boasting of seasoned experts equipped with extensive knowledge and experience, is committed to ensuring your MLOps journey is smooth, strategic, and successful.\n\nWhy trust Tryolabs? Let's delve into what makes us stand out:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 2fa5ab43-8fce-4cd1-ae78-1d55602a7a24
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Why trust Tryolabs? Let's delve into what makes us stand out:\n\nExpertise: Our team comprises seasoned professionals who have honed their skills over years of practice. Not only experts in MLOps, but also in various aspects of Machine Learning, and data science. Experience: With a rich and diverse portfolio of complex MLOps projects under our belt, we are adept at navigating and addressing varied challenges. Strategic insight: We don't just execute projects, we strategize. We take the time to understand your business, goals, and specific needs, allowing us to provide you with the most effective and efficient solutions."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 2b1122bf-cbe7-4a6b-b64a-19f036b38716
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "As your MLOps partner, Tryolabs brings deep expertise, real-world experience, and commitment to guide you every step of the way. We understand the complexities of MLOps and have the solutions to help you succeed.\n\nReady to start your MLOps journey? Here are 3 ways we can help:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: ab19d8d2-049f-4cf5-94d8-7f0533ed42f4
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "Immerse yourself in our comprehensive white paper on MLOps. This guide provides actionable strategies on team building, governance, and more. It's the roadmap you need to achieve data-driven excellence. Download the Tryolabs MLOps White Paper now. Have questions? Want hands-on guidance? Our experts are ready to advise and accompany you. Reach out to us at hello@tryolabs.com and we'll help make your MLOps implementation smooth and successful. Bring the learning to your team with a customized MLOps Talk. Our in-house sessions deliver practical insights to skill up your staff. Book an AI Talk today and let's explore MLOps together."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 3145e242-619f-4a89-90dc-d42021666b90
encoding: FLOAT32
metadata:
{"title": "MLOps: The strategic compass for AI-driven enterprises", "link": "https://tryolabs.com/blog/mlops-the-strategic-compass-for-ai-driven-enterprises", "language": "en", "description": "Discover the pivotal role of MLOps in AI-driven organizations. Dive deep into its significance, learn how it bridges AI dynamics with business objectives, and explore key stages to optimize AI deployments for tangible results. Embark on the MLOps journey with us", "chroma:document": "With Tryolabs as your guide, you don't have to navigate the MLOps path alone. Take the first step now and let our experts help drive your success. The future of AI is bright when we work together!"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: b51f2929-f7a0-49e7-a1fc-c83aca4fc8c6
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "If you\u2019re new to the topic of market simulations, we highly recommend reading the first blog in our series to get familiar with the basics. In that post, we briefly introduced the concepts of demand forecasting and pricing optimization. We also explained why this tool is critical to providing the best pricing solutions. Moreover, we took a deep dive into the inner workings of our simulation process and the mathematical definition of our demand function. Finally, we used the simulator to test a learning model capable of making pricing decisions and learning from them within a market scenario."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 9eed4e3f-2f35-4a05-9363-716bfa6e07c8
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "This second post will explore more complex market scenarios to test and evaluate a learning model. We\u2019ll mainly focus on balancing exploration and exploitation while optimizing. We\u2019ll further discuss how adding Bootstrap to the learning model\u2019s output enables that balance."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 71458db5-0698-4149-99ad-7c2123d58397
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "In this section, we'll quickly review our market simulator's building blocks. If you're already familiar with it, feel free to skip to the next section. As a starting point, let's revisit the mathematical definitions that determine our market. The first definition we want to review is the underlying expected demand Q \u02c9 \\bar{Q} Q\u02c9\u200b, a function of price and, optionally, other exogenous factors. Furthermore, once we define our expected demand, we can also calculate the expected profit function as a function of the expected demand, the price, and the given cost. It's also important to remember that our historical data is modeled by sampling a Poisson distribution, using Q \u02c9 \\bar{Q} Q\u02c9\u200b as the"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 0200c496-b9f9-4b67-b8cb-74ad281c1939
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "the given cost. It's also important to remember that our historical data is modeled by sampling a Poisson distribution, using Q \u02c9 \\bar{Q} Q\u02c9\u200b as the corresponding \u03bb \\lambda \u03bb for each price."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 4888d7ea-d16c-4250-8b49-904173f98eff
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Figure 1. a) Three important equations to keep in mind, defining, from top down: expected demand, expected profit, and realized demand. b) The expected demand curve is represented in blue, the expected profit is represented in red, the realized demand can be seen as the green dots, and the true optimum profit is shown as a blue dot."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 9a7b50e6-d7bd-494b-872e-9bf41363c3a6
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "We will also briefly reiterate how the simulation works. It's a dynamic, fixed-duration timestep (think daily) market simulation where we use all the historical data available to train a specific demand forecasting model at each step. Once we obtain the values for the exogenous factors, this trained model will allow us to forecast the expected demand ( Q \u02c9 \u2217 \\bar{Q}^* Q\u02c9\u200b\u2217) for any price. These exogenous factors are a part of the market state, so the simulator actually produces their values. With the model estimating Q \u02c9 \u2217 \\bar{Q}^* Q\u02c9\u200b\u2217 and the simulated exogenous factors X X X, we can define a price range (the x-axis in the plot) and obtain our forecasted demand and profit. By maximizing"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 7f673397-b004-4687-ad59-d58b627a1457
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "simulated exogenous factors X X X, we can define a price range (the x-axis in the plot) and obtain our forecasted demand and profit. By maximizing that forecasted profit, we find our optimum price. Finally, the optimum price is set in the market, and the (simulated) market responds with its corresponding realized sales units, generating a new data point for our dataset in the process. The simulation now moves to the next time step and repeats the same process as before. In the following figure, we can see the same simulation animation as in the previous blog."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d9a48376-ded6-45bc-b018-7c17d927da60
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Figure 2. Animation of a market simulation loop with up to 350 iteration steps.\n\nIn our previous post, we discussed the assembly of the market simulator and modeled a single-item market scenario with a specific expected demand curve. By setting the market conditions, we were able to see how well the learning model, XGBoost, predicted the underlying expected demand. The simulator allowed us to understand the model's strengths and weaknesses since we knew precisely what the objective demand function was. Figure 2 illustrates that the model learned how to provide the optimal price in the given situation, as the suggested optimum further approached the true optimum with each iteration."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 7bc2e842-d36e-46d9-a9bf-69ae813aa391
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Upping the game\n\nOur last blog post also showed how XGBoost successfully handled a pricing optimization problem given a particular scenario with fast sales in our market simulation. We now want to see how the model fares in other, more challenging scenarios.\n\nVersatility is key"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 142de86c-512c-4e26-9c92-6d003e82aa80
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Versatility is key\n\nWe designed the demand function\u2019s definition with flexibility in mind. So modeling various demand curves is simply a matter of adjusting the values for \u03b1 , \u03b2 {\\alpha}, {\\beta} \u03b1,\u03b2, and \u03b3 {\\gamma} \u03b3. By doing so, we get to model different conditions of interest. We can change the speed at which our simulated item can sell, the location of the elastic region of the demand, and how steep this region is. We can also model both perishable and non-perishable products."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: ae155ec6-bd60-424a-ac48-3cf34626bed8
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "In our previous post, we worked with a demand function that described a perishable product with a high daily sales volume that would saturate at lower prices, seeing no sales increase after a specific low price was reached.\n\n\n\nThat means that no further price reduction will impact the number of units you sell after hitting a certain price. This demand curve is displayed on the right in Figure 3a. Figure 3.a) Fast sales saturating demand curve.\n\nAnother demand function we were interested in testing out is one with both slow sales and the previously explained market saturation condition."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 26307787-035c-49d1-889c-c3d368656f9f
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Another demand function we were interested in testing out is one with both slow sales and the previously explained market saturation condition.\n\n\n\nThis slow demand curve is shown in Figure 3b. Notice that the y-axis is much smaller now. In particular, the expected demand, shows sales of less than 1 unit per period.\n\n\n\nThis type of demand is often referred to as 'sparse' because sales don't occur at every time step and the item's natural sales velocity is slow. Figure 3.b) Sparse sales saturating demand curve."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 45feac9d-5d71-43b3-a618-8e700e18d820
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "The last demand function we wanted to try out exhibits an exponentially decaying pattern, as shown in Figure 3c. With this particular model, we observe a continual surge in sales with every reduction in price, and the saturation effect is no longer present.\n\n\n\nThis type of demand model corresponds to a non-perishable product that the market would stockpile as the price decreases. Figure 3.c) Exponential demand curve.\n\nA bit of history"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 33635abe-a579-4547-a900-64ecdd44cff9
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "A bit of history\n\nNow we have three distinct scenarios that we want to test our model on. But we also wanted to understand the impact of the historical data's price placement on our model's ability to learn. Figure 4 clearly shows what we mean by this. We have four different historical datasets that vary in where the historically tried prices are located. We've analyzed some simulations with a significant standard deviation in historical prices, but our experience has shown that this behavior is rather uncommon in real-life datasets. So we won't discuss those simulations here.\n\nFigure 4. Different demand function and historical dataset combination scenarios to evaluate our model in."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: a171a420-17a9-4d20-9ae2-dc0eb9c5c804
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Figure 4. Different demand function and historical dataset combination scenarios to evaluate our model in.\n\nPutting it together\n\nNow that we have different demand functions and historical datasets to try, we\u2019ll combine them into the nine situations in Figure 5. Given those market realities, we\u2019ll be testing how good our demand forecasting model is at predicting the expected demand in the daily repricing situation of our simulator. We\u2019ll also carefully analyze if the lack of exploration becomes an insurmountable problem for our model in any scenario.\n\nDifferent historical datasets we will use to combine with our fast sales demand function to evaluate our model in."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 78227d2b-7010-4ded-b836-53ea67f594af
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Different historical datasets we will use to combine with our fast sales demand function to evaluate our model in.\n\nDifferent historical datasets we will use to combine with our slow sales demand function to evaluate our model in.\n\nLow price historical dataset exponential demand function scenario to evaluate our model in."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 4e4ff419-4b52-4b9b-be5f-2c757017b135
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Low price historical dataset exponential demand function scenario to evaluate our model in.\n\nOnce we run all the pertinent simulations with the configurations mentioned above, we can see that the proposed demand forecasting model had some issues. In particular, all the scenarios with sparse sales were performing poorly, as shown in the summary in Figure 6. In the displayed table, we can see two metrics to consider in determining whether or not the strategy used was successful in a given scenario. These measures correspond to the percentage of cumulative profit losses concerning the actual expected optimum incremental profit and the rate of steps with less than 5% of profit losses."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 852e3d28-b9ee-4382-9574-bf0b91d82509
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "The first value was introduced in the first blog post of this series and was calculated as the percentual difference between the accumulated expected profit, had we always used the accurate optimum price, and the accumulated profit made using the predicted optimum price. A perfect score here would be 0, indicating we lost no profit even without knowing the actual demand curve.\n\nThe second measure we consider is the percentage of iteration steps in which we experienced a loss of less than 5% of the total achievable profit for that step. In this case, a perfect score would be 100% indicating all steps lost less than 5% of the attainable profit in each step."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6e90a055-bcfd-4d74-b853-1369e15f27c6
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Ideally, we want our strategy to minimize the first value mentioned and maximize the second. We use these values to evaluate our strategy's overall success in the different scenarios. As you can see, it's not performing at its best at the moment\u2026\n\nFigure 6. Results for the different demand and historical data combined scenarios.\n\nBut one of the advantages of having the simulator is that we can now inspect model behavior in a controlled environment and, knowing what should have happened, understand where the model went astray."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 0a38913a-033c-4176-8e04-1ffac89dedf9
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "We'll now zoom into one of the unsuccessful scenarios to determine what problems we can identify. The situation in Figure 7 corresponds to a market with sparse sales and historical data located just below the optimum price. Looking at the animation, we understand why this scenario is so demanding for the learning model. In Figure 7, we see that sparse sales create a 0 inflation problem. That means that we're going to have (price, sales) points with 0 sales for two very different reasons: a) because the price is too high for there to be any sales, and b) the product hasn't sold because not enough time has passed yet. That creates a challenging training set because the model can't distinguish"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 55287780-31a0-4ae1-a105-2aee67f39281
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "and b) the product hasn't sold because not enough time has passed yet. That creates a challenging training set because the model can't distinguish between these two situations, and because we optimize for maximum profit, that region is of very little interest because its forecasted profit is rather low. As that price region isn't of interest, the model has very little incentive to explore those prices, failing to improve its estimates, thus missing the optimum price entirely."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: f85b87df-c9e9-474f-989e-c1506231c0ba
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Figure 7. Market simulation of a sparse type of demand and historical prices situated below the known optimum price.\n\nWe exemplified the problem with the situation shown in Figure 7, but all the failing scenarios present variations of the same problem. Fortunately, this isn't an unknown phenomenon. This is a widely known problem in Reinforcement Learning and is known as the Exploration-Exploitation dilemma.\n\nIssues with Exploration & Exploitation"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: a6889944-cb9b-452e-b699-9e97818f8ebd
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "To ensure we identified the problem correctly, we designed an experiment on the other edge of the spectrum. Instead of doing pure exploitation, we\u2019ll go down the road of pure exploration. This is not what we recommend in actual production, as price exploration has a cost, but as a theoretical experiment, it\u2019s helpful to understand the characteristics of these approaches. So, instead of optimizing for the maximum forecasted profit, we built an agent that randomly chooses a new price from the entire price range. This strategy would undoubtedly force exploration and obtain better price space coverage. A more extensive coverage should enable the model to better grasp the expected demand curve."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: dc2e19ea-0d32-4971-a484-b328e9773a40
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "exploration and obtain better price space coverage. A more extensive coverage should enable the model to better grasp the expected demand curve. So we tried it, and voila! As you can see in Figure 8, exploration allows the model to improve it's estimates of the demand curve across the entire price range. The corresponding animation shows that the light blue curve is very similar to the blue curve and as time advances it becomes more and more so."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 3095845e-284b-4b86-873f-a161a5bf6f56
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Figure 8. Animation of the same scenario as Figure 7 but with a strategy in which a random price is chosen at each time step.\n\nWe needed to implement a sound exploration approach into our modeling strategy that would allow us to understand the demand curve in the regions where we have no data, but that's also efficient and doesn't waste time exploring prices that produce fewer gains or losses. And this exploration approach should work across all scenarios. Considering this problem and discussing potential techniques, we found many similarities with the Multi-Armed Bandit (MAB) problem."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 265ca4b6-7c05-45c9-9eac-87096336bbe3
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "As a brief overview, the MAB problem involves finding the best resource allocation strategy among multiple options with uncertain outcomes. To exemplify, say you have a set of slot machines, each giving a different payout. You want to determine through trial and error which slot machine to pull to maximize your gains. Typically, one would do this by pulling each slot machine's lever multiple times and building an estimate of the distribution of the expected reward of each try. After pulling each lever, you update the distribution estimates and pick the slot machine with the highest expected payout. You can quickly see that this strategy will have the same exploration problem as we do in our"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 5ac296d0-22b0-43eb-a152-a89f6fd2cd1c
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "pick the slot machine with the highest expected payout. You can quickly see that this strategy will have the same exploration problem as we do in our simulator. If we happen to receive higher samples from a slot machine that initially has a lower expected payout, we will continue to use that one and refrain from testing the others."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: be0679fd-eaac-4ea2-a0e3-0df03b583bdc
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "We can see this exemplified in Figure 9. The second slot machine had the largest expected payout when the image was taken. But if you look at the actual payouts, you can see that the first slot machine will give the largest payout. One popular way to solve this problem is the Upper Confidence Bound algorithm.\n\nFigure 9. Four different slot machines with their corresponding expected payouts in blue, true payouts in red, and their confidence intervals."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d1a490c2-f58b-4f98-96f5-1059cd738abe
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "The Upper Confidence Bound algorithm is based on the principle of Optimism in the Face of Uncertainty. Before we explain that, it's important to recall that the sample mean estimator has a particular variance that depends on the sample size. The more we pull on a specific slot machine's lever, the more we can trust our sample mean estimator. In the image above, this translates to smaller confidence bands around the expected result as the confidence interval is reduced. Optimism in the Face of Uncertainty results from choosing the lever with the highest payout + uncertainty. In practical terms, this means we take the upper bound of the box we can see in Figure 9 instead of the hightop-most"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 684e8da5-4622-4ecf-a523-99758ff356ad
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "highest payout + uncertainty. In practical terms, this means we take the upper bound of the box we can see in Figure 9 instead of the hightop-most blue line."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: afc9bb57-6577-47d9-ac00-b35a4421f93e
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "If the newly chosen lever has a bigger payout, its distribution will shift upwards and gradually increase its likelihood of being picked. If it turns out we were wrong and that slot machine pays less, then the sample mean's variance will decrease, making the box smaller and the lever less likely to be picked. By rewarding pulling on levers with high variance, we encourage exploration and the discovery of new possibilities."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: f0a65951-0201-4bb5-803a-682503609f7f
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "That sounds like a great idea, but there are a few caveats, starting with the exogenous factors. Usually, you would use the number of times a price has been tried in the past to determine sample variance, but we have multiple factors contributing to demand. It's not simply a question of having tried a particular price but also about the context in which it was tested for a specific factor. Another issue is that the price is continuous, just as the exogenous factors could be.\n\nThe question then becomes: can we build a confidence interval for our forecasted expected demand that depends on both covariates and a continuous price?"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 882e7d96-e668-4195-b66b-ddff62451d37
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "We can answer in the affirmative thanks to a widely known technique from the field of statistics: Bootstrap.\n\nBootstrap"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 203f868b-18b6-4a7f-af5f-b53ad36c0c18
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Bootstrap is a method that allows you to obtain an estimate for any distribution parameter from a statistic whose values you can observe. This strategy works by resampling the original data with replacement, producing several Bootstrap samples. You then calculate the statistic of interest for the bootstrapped samples and use those to get a feel for the uncertainty of the parameter of interest. If we consider our model's forecasts as a Random Variable that depends on the samples we got in our training set, we can estimate the distribution of these targets, i.e. the forecasted expected demand, using Bootstrap. Namely, you'll obtain a high quantile of the distribution of your forecasted"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 1ffd5e5d-da65-4c54-9a64-047272fcf5d5
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "of these targets, i.e. the forecasted expected demand, using Bootstrap. Namely, you'll obtain a high quantile of the distribution of your forecasted expected demand (can you see where this is going?)."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 7ced9cec-01ab-40ae-9aec-42da8cd7cfb2
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Figure 10. Bootstrap sampling strategy used.\n\nThe strategy for bootstrapping the output of a machine-learning model is displayed in Figure 10. It involves training a model with each Bootstrap sample, getting the forecasted expected demands from each model, and then using all the different forecasts as their distribution estimate. We then use a high quantile to calculate the upper confidence bound. In our case, we chose the 0.8 quantile. Figure 11 shows this strategy aiding our model to explore and converge to the optimum price.\n\nFigure 11. Animation of the same scenario as Figure 7 but using Bootstrap to force exploration."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 587b0a4b-5ea6-460e-a57e-0076f1e8e6a7
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "We trained 30 XGBoost models for this simulation, obtaining 30 different forecasts for each price point, following the Bootstrap technique described before. If you look at the plot above, you can see we added a few shaded areas with boundaries corresponding to several quantiles (50%, 60%, 70%, 80%, 90%, and 100% quantiles) of the Bootstrap estimated distributions. As expected, the bands are wider at the extremes where we have fewer samples. Because observations are less frequent there, we have fewer chances of one ending up in the Bootstrap samples, hence making forecasts less stable and allowing us to quantify uncertainty. This process enhances the model's ability to make predictions in"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 20924585-21fd-4933-9723-ee1fe499067c
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "samples, hence making forecasts less stable and allowing us to quantify uncertainty. This process enhances the model's ability to make predictions in areas with limited data and also enables us to measure the level of uncertainty associated with those predictions."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: cd95107c-f403-42cf-9430-592295fc7265
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Let's be optimistic and optimize for the 80% quantile of the forecasted demand (and hence for profit since the profit function is a linear function), achieving the exploration effect we want. Looking at the above, it's easy to see that the price dispersion the model is trying is much higher now. The model tests out different sections of the demand curve and learns from them, eventually discovering the optimum price. Also, it's not trying prices at random. It's exploring prices that make sense. And most of the time, the prices that produce profits are very close to the optimum."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: de1473a7-8c87-43df-91dc-8ca393b983ea
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Side note: If you paid attention to the simulation\u2019s animations, you might have noticed that the expected demands look pretty smooth compared to the usual XGBoost output. Don\u2019t worry; your eyes aren\u2019t deceiving you. We just found XGBoost\u2019s output a bit too angular for a demand curve, so we smoothed it out a little."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: ba426cb6-281e-4215-a783-adc94bb45559
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Choosing the number of models you\u2019ll be bootstrapping is not trivial. If you use too few, you may not get enough exploration, but it will be less computationally expensive. On the other hand, the more models you use, the better the results, but the computational costs increase proportionally. You should keep this in mind when using Bootstrap for your projects. We recommend trying different quantities of models to determine the best trade-off between time, resources, and the results obtained for your project(s)."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: bdd896ab-de7a-4911-9b62-d7430ad0b17c
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "We used this strategy in all the market simulations we mentioned earlier, as shown in Figure 12. Pretty awesome, right? All these scenarios are played out in parallel for multiple learning agent candidates. This allows us to observe all these variant models' behavior and understand their strengths and weaknesses. With this approach, we can not only select the most effective model but also identify any issues with its performance to make necessary improvements. This leads to quicker R&D cycles and more efficient development.\n\nFigure 12. Gif with all the scenarios playing at once."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: e43d96f2-6648-4749-bc1f-6c6716807323
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Figure 12. Gif with all the scenarios playing at once.\n\nTo emphasize the findings, we reconstructed the chart we presented earlier with the updated results, as illustrated in Figure 13. We can see that the model reaches the optimum price across all simulation scenarios now. The difference from the results shown in Figure 6 is massive. And we even managed to limit the percentage of cumulative profit losses to less than 5% in all cases!\n\nFigure 13. Results for the different demand and historical data combined scenarios.\n\nWrapping up"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6be01c93-8602-4e9a-bbf8-92cd7fe6a3ef
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Figure 13. Results for the different demand and historical data combined scenarios.\n\nWrapping up\n\nIf you made it this far, thank you! It shows you're as interested in these pricing problems as we are! Let's provide a brief recap of everything we've learned so far.\n\nPricing pitfalls in live production can be difficult to anticipate without evaluation. A market simulator can help by testing scenarios and bringing potential problems to the surface for better decision-making."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 81baca72-2625-4bb2-b0ac-c03d690ae185
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Determining the optimal price point for a market will involve exploration. It's impossible to predict the outcome of a price range without testing. It's critical to try out different options to better understand potential outcomes.\n\nUsing statistical techniques like bootstrapping maximizes data utility and quantifies uncertainty, leading to smarter explorations and informed pricing decisions.\n\nWe were able to validate the above claims across a varied range of scenarios using the simulator, ensuring that our models are better prepared for production use and maximizing our clients' success.\n\nFinal thoughts"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 457354b8-1558-4c68-b6e4-ac0997eefae1
encoding: FLOAT32
metadata:
{"title": "Mastering pricing strategies vol 2: Exploring complex market scenarios", "link": "https://tryolabs.com/blog/2023/05/19/mastering-pricing-strategies-vol-2-exploring-complex-market-scenarios", "language": "en", "description": "Explore complex scenarios with our advanced Market Simulator in this post. Learn how to balance exploration and exploitation with Bootstrap integration for improved market adaptability.", "chroma:document": "Final thoughts\n\nAlthough Bootstrap played a crucial role in our approach and ensured that all scenarios converged, the pace of convergence was not quite up to our standards - it was a bit slow for our tastes. To achieve the final results above, we tweaked our pricing agent even further - particularly regarding its approach to unknown regions.\n\nIf you're interested in getting more information about these developments, whether you have a specific market scenario you'd like to test out using our simulator or if you're interested in evaluating a particular model, we'd be delighted if you got in touch with us."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 90ac58d7-caa9-4b20-8667-4ffa0228a1e0
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "If you've ever worked with tabular data, you probably know the process: import the data into pandas, clean and transform it, and use it as input for your models. But when the time comes to scale up and take your code to production, there's a good chance your pandas pipeline starts to break down and runs slowly.\n\nBut there's no need to panic! Given our extensive experience handling large datasets and using pandas in over 80% of our projects, Tryolabs has gained valuable insights into making your pandas code run faster.\n\nTip 1: Vectorize like there\u2019s no tomorrow. Using vectorized operations eliminates the need to manually iterate through rows."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: a6ee3b3b-928e-499c-a0dd-73fcf49f2b7e
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Tip 1: Vectorize like there\u2019s no tomorrow. Using vectorized operations eliminates the need to manually iterate through rows.\n\nTip 2: Iterate like a semi-pro. When vectorization is impossible, consider looping over the data frame using methods such as list comprehension with zip , lru_cache or parapply .\n\nTip 3: Embrace NumPy. To enhance performance, consider working directly with the Numpy array through .to_numpy() ; it can provide a significant bump in speed.\n\nTip 4: Know your dtypes. Selecting the appropriate dtype is critical for memory and computation optimization."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 23c392c1-433d-4006-a5dc-6fd288d3a292
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Tip 4: Know your dtypes. Selecting the appropriate dtype is critical for memory and computation optimization.\n\nTip 5: Long-term storage. Pandas can read many formats such as CSV, parquet, pickle, JSON, Excel, etc. We recommended using the parquet format, a compressed, efficient columnar data representation.\n\nWe'll also explain what can slow your pandas down and share a few bonus tips surrounding caching and parallelization. Keep reading, and you'll become a pandas pro in no time.\n\nPanda brought to life by the power of Midjourney."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 87bae783-e2ac-4d70-a3b5-a301f3bd17e1
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Panda brought to life by the power of Midjourney.\n\nThe best way to demonstrate the value of our recommendations is to test the techniques with real datasets. If you are curious about the specifics, take a look at the following section; otherwise, feel free to skip it.\n\nBenchmarking setup We conducted the benchmarks in the following sections on two anonymized data frames from actual projects. And we've made those datasets available to share. Download the data frames to run the tests yourself. Data frame 1"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 396ef85c-5c4c-449c-aaea-73666dc579a9
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Data frame 2 Please note that depending on your computer's specifications, you may have trouble opening the data frames. The creator of pandas, Wes McKinney, stated that as a rule of thumb, it is recommended to have 5 to 10 times the amount of RAM as the dataset size. To avoid these issues in running the benchmarks, we used a high-performance desktop computer with a 10-core/20-thread i9 10900K, 128GB of RAM, and a fast SSD running Ubuntu 20.04. Regarding the software used, we chose Python 3.10, pandas 1.4.2, and pyarrow 8.0.0.\n\nLet\u2019s start with the best tip we have to share.\n\nTip 1: Vectorize like there\u2019s no tomorrow"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 95303187-5d20-468a-8265-cade3516cbb6
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Let\u2019s start with the best tip we have to share.\n\nTip 1: Vectorize like there\u2019s no tomorrow\n\nVectorized operations in pandas allow you to manipulate entire data frame columns using just a few lines of code rather than manually looping through each row individually."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 3a24ca70-115f-4d03-a741-3b4d5999f00a
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "The importance of vectorization cannot be overstated, and it's a fundamental aspect of working with pandas. It's so crucial, in fact, that we even created a \"catchy\" rhyme to remind ourselves of its importance: \"Hector Vector is a better mentor than Luke the for loop.\" In all seriousness, however, pandas lives and breathes vector operations, as should you. We strongly recommend you embrace vectorization and make it a core part of your pandas workflow.\n\nHow does it work?\n\nA key element of vectorized operations is broadcasting shapes. Broadcasting allows you to manipulate objects of different shapes intuitively. Examples are shown below."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 5e61a10f-4617-42a3-bbd7-3a2651f20f02
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Array a with 3 elements is multiplied with scalar b which results in an array of the same shape as a Source Array a of shape (4, 1) is added with array b (3,) resulting in an array of shape (4, 3). Source\n\nMuch has been written about this (including these articles about NumPy Optimization and Array Programming with NumPy), and it's essential in deep learning, where massive matrix multiplications happen all the time. But we'll limit ourselves to 2 short examples.\n\nFirst, imagine you want to count a given integer's occurrences in a column. Below are 2 possible ways of doing it.\n\nIn testing df1 and df2 , we got a speedup of just 82x by using the count_vectorized method over the count_loop ."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: dc4b1485-b8b6-468a-8ae4-810e7f99411a
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "In testing df1 and df2 , we got a speedup of just 82x by using the count_vectorized method over the count_loop .\n\nNow say you have a DataFrame with a date column and want to offset it by a given number of days. Below, you\u2019ll find two ways of doing that. Can you guess the speedup factor of the vectorized operation?\n\nBy using vectorized operations rather than loops for this costly operation, we got an average speedup of 460x!\n\nOne small note on the offset_loop function: we construct d outside the list comprehension. Otherwise, it would get built in each iteration, which would double its runtime (this is noteworthy in its own right)."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6c11fb90-12fa-461f-b6de-63296ab56bf9
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "But you get the point, vectorized operations in actual data are a lifesaver. Remember, it isn't the few microseconds you can shave off a single operation that counts; the total time saved over multiple operations makes the difference. This can take a pipeline from \"impossible to run\" to \"runs in a reasonable amount of time.\u201d\n\nNow that you know how vectorized operations can significantly improve your code's performance, it's time to implement it. Look at your current projects and see where you can apply this technique. You may be surprised at how much of a difference it can make in speed and efficiency."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 8af05efa-3900-4ad8-8df2-adc0db7fca65
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Although vectorized operations in pandas are highly recommended for optimal performance and efficiency, it's important to note that not all operations can be vectorized, and there may be instances where alternative methods, such as iteration, will be necessary.\n\nTip 2: Iterate like a semi-pro\n\nYou might be wondering why you\u2019re not able to iterate like a fully-fledged pro. The answer is simple: pros vectorize and rookies iterate.\n\nWe know it\u2019s not that simple in practice. You can\u2019t vectorize all operations all the time, but hear us out: whenever you can, vectorize ruthlessly; you\u2019ll thank yourself later."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 8b75ec7b-f0c9-43c3-8ec3-233c37059e0d
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "There will be times when there\u2019s no alternative to looping over the millions of rows in a data frame. You may choose to perform these complex operations yourself or solicit help from an external service. Whatever the case, we\u2019ll go through various iteration methods.\n\nKey takeaways: Vectorize if possible. Use a list comprehension with zip . Look into lru_cache if your columns have several repeats. Look into parapply .\n\nThe benchmark\n\nWe\u2019ll apply the following function to the datasets:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: cc04739f-32ae-4290-af24-c873d7c4d075
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "The benchmark\n\nWe\u2019ll apply the following function to the datasets:\n\nIt removes all the words appearing in the phrase words_to_remove from the given remove_from phrase and eliminates HTML tags while keeping the words with a length greater than or equal to min_include_word_length .\n\nThink of remove_from as a long description from which you want to remove some words that already appear in another data column.\n\nFor instance, say you have the following data frame:\n\nApplying remove_words to all rows would result in the following list:\n\nfor loops\n\nThe first and most intuitive way to iterate would be to use a Python for loop."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: b426c613-379f-467d-b709-3139fe8cf55b
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "for loops\n\nThe first and most intuitive way to iterate would be to use a Python for loop.\n\nThere\u2019s overhead everywhere: accessing single values of df and dynamically creating a list with append mean this will be the slowest method and we\u2019ll use it as our baseline for comparison. But you want to avoid this at all costs.\n\napply\n\nAn easy next step would be to use pandas' apply method, which handles the looping internally. You\u2019d expect this to be blazing fast, but it\u2019s only about 1.9x faster than baseline. The code is below. But because better methods are readily available, you should avoid this as well."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 9c03cc0e-24ad-40c4-a5b7-0ae49c9bf505
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "In each iteration of df.apply, the provided callable gets a Series whose index is df.columns and whose values are the row\u2019s. This means that pandas has to generate that series in each loop, which is costly. To cut the costs, it\u2019s better to call apply on the subset of df you know you\u2019ll use, like so:\n\nThis simple tweak makes the operation 2.1x faster than the baseline, but we\u2019d still recommend avoiding this method.\n\nlist-comp + itertuples\n\nUsing itertuples combined with the beloved list comprehension to iterate is definitely better. itertuples yields (named) tuples with the data on the rows. Notice that we subset df again. This is, on average, 4.6x faster than baseline.\n\nlist-comp + zip"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6f694197-c2fe-4c03-b84e-2d9440c9fc4c
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "list-comp + zip\n\nA slightly different approach that yields about the same performance (4.6x faster than baseline) is to use a list comprehension again but iterate over the desired columns with zip. zip takes iterables and yields tuples where the i-th tuple has the i-th element of all the given iterables in order. This approach looks like this:\n\nlist-comp + to_dict\n\nA slower variant of this approach is to use to_dict(orient=\"records\") to iterate over dicts of rows of the dataset. This is about 3.9x faster than the baseline. It was achieved using the following code:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 5ee36dc1-c5cb-4c0a-87ee-8f9452245e23
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "This is not a hard rule but, from observation, we determined that using to_dict almost tripled the memory footprint of our program, so be judicious when using it.\n\nBonus method 1: caching"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: c4a0d526-0959-4e58-9a5c-53b719132222
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "In addition to the iteration techniques we've discussed, two other methods can help improve your code's performance: caching and parallelization. Caching can be particularly useful if your pandas function is called multiple times with the same arguments. For example, if remove_words is applied to a dataset with many repeated values, you can use functools.lru_cache to store the results of the function and avoid recalculating them each time. To use lru_cache , simply add the @lru_cache decorator to the declaration of remove_words , then use your preferred iteration method to apply the function to your dataset. This can significantly improve your code's speed and efficiency. Take, for example,"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 03d7c050-beac-4eef-aaee-8321cd642229
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "preferred iteration method to apply the function to your dataset. This can significantly improve your code's speed and efficiency. Take, for example, the following code:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: ac95d1f9-484f-46f7-a773-4bdfc5225774
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Adding this decorator produces a function that \"remembers\" the output for inputs it previously encountered, removing the need to run all the code again. You can read more about it in Python's official documentation here. Don\u2019t forget to play with the maxsize parameter to explore the speed vs memory tradeoff.\n\nYou may notice that the cached function\u2019s performance for df1 is similar to its non-cached counterpart. This goes to show that caching isn\u2019t the be-all and end-all. In our case, df1 's structure doesn\u2019t have as much repetition, so caching doesn\u2019t help as much as it does fo df2 where the speedup is 46x.\n\nBonus method 2: parallelization"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 35cd0052-273c-4947-bb9d-f13d60e1f7e6
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Bonus method 2: parallelization\n\nThe last ace up our sleeve is using pandarallel to parallelize our function calls across multiple independent chunks of df. The tool is easy to use: you simply import and initialize it and then change all your .applys for .parallel_applys.\n\nIn our case, we measured an 12-fold improvement.\n\nHowever powerful, parallel_apply is no silver bullet. It pickles each chunk of the df into /dev/shm (which is effectively RAM); therefore, you might run out of memory!\n\nIf this happens, you can limit pandarallel 's memory usage by initializing it with fewer processes, which will also impact performance."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 3a195084-cd39-465d-ae23-94e883cc98aa
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "If this happens, you can limit pandarallel 's memory usage by initializing it with fewer processes, which will also impact performance.\n\nWhy not combine both? Well, we tried and saw no practical benefit. It either went as fast as bare parallel_apply or slower than a bare list comprehension with cache. This might have something to do with how parallelization is achieved and the fact that the cache might not be shared. However, we would need to pursue our investigation further to draw any firm conclusions.\n\nResults\n\nA summary of the results is shown below:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6b882344-751e-4484-a345-77d301f9dd8c
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Results\n\nA summary of the results is shown below:\n\nDescription df1 df1 df2 df2 Description Time [s] Speedup Time [s] Speedup loop 546.6 1 245.9 1 apply 289.3 1.9 135.7 1.8 apply (only used cols) 263.9 2.1 121.7 2.0 itertuples (only used cols) 116.1 4.7 54.7 4.5 zip (only used cols) 115.6 4.7 54.2 4.5 to_dict (only used cols) 137.8 4.0 64.0 3.8 cached zip (only used cols) 118.1 4.6 5.3 46.4 parapply (only used cols) 44.5 12.3 19.3 12.7 cached parapply (only used cols) 45.5 12.0 10.6 23.2\n\nRemember to use the above tips to optimize performance when iterating with pandas in the future.\n\nTip 3: Embrace NumPy"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 779c5e38-5e2d-42e5-9549-4b55633187aa
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Remember to use the above tips to optimize performance when iterating with pandas in the future.\n\nTip 3: Embrace NumPy\n\nPandas is built on top of NumPy, which is known for its performance and speed in handling large arrays and matrices of numerical data. This helps make pandas efficient and fast when working with large datasets. Now that you\u2019re an expert in vectorizing operations embracing numpy is a natural next step. This section is short and sweet: if you need a bit of extra performance, consider going down to numpy by using .to_numpy() and the numpy methods directly."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: b1e5922d-f089-4ef9-82f4-427a00bc5660
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Benchmarking this method is close to impossible. Results will vary wildly between the use case and the implementation details of your code. In our tests, we observed speedups from anywhere between 2x to an absurd 3000x.\n\nContinuing with the above examples, what if we tried counting or offsetting directly in numpy? Have a look at the following code:\n\nAs you can see, it\u2019s easy to accomplish: add .to_numpy() and use numpy objects. Regarding the count, this gives a total 185x speedup (2.2x over the vectorized pandas method). In regards to the offset, this gives a whopping 1200x speedup over the loop operation."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 194acbce-ab5c-4082-b846-79fa29f4ad37
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "We hear you! These numbers are somewhat artificial because no one in their right mind would offset a day column like this. But this may happen to beginners familiar with other programming languages, and these examples should serve as a cautionary tale.\n\nThese speedups are made possible by compounding the benefits of vectorization and the direct use of numpy. This also eliminates the overhead added by pandas' killer feature set."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 2fd1ca1e-9efb-4cce-8948-06a8d212ca3a
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Be warned that sometimes this isn\u2019t as simple as placing .to_numpy() after each series in your operation. Case in point: a team member reported that using .to_numpy() in datetime data with timezones actually dropped the timezone. This should be fine if all the data is at the same timezone and you\u2019re calculating differences. Still, it\u2019s something to keep in mind."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: fea6c34c-6b81-48b1-b88a-4719a73e184a
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Another team member experienced some weirdness with dates as well. When calling .to_numpy() on a Series of date times, they would get an array of datetime64[ns] . However, when calling .to_numpy() on a single pd.Timestamp , an int was returned. Hence, trying to subtract a single date from an array of dates will return gibberish unless you do .to_numpy().astype(int) .\n\nGiven these speedup factors and despite its caveats, you want this tool in your arsenal, and you shouldn't hesitate to use it.\n\nTip 4: Know your dtypes"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 93f7259d-0da1-44b4-9b32-1a4244bfb63a
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Tip 4: Know your dtypes\n\nIn pandas DataFrames, the dtype is a critical attribute that specifies the data type for each column. Therefore, selecting the appropriate dtype for each column in a DataFrame is key.\n\nOn the one hand, we can downcast numerics into types that use fewer bits to save memory. Conversely we can use specialized types for specific data that will reduce memory costs and optimize computation by orders of magnitude.\n\nWe\u2019ll talk about some of the most prevalent types in pandas, like int , float , bool , and strings. But first, here\u2019s a a primer on the dreaded object .\n\nThe object type"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 2b58e6df-eafc-4f65-9481-0d514ca0fe68
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "The object type\n\nRegarding datatypes, pandas has many types that efficiently map to numpy types at the fast C level. However, when there\u2019s no easy mapping, it falls back on python objects. For the uninitiated, object is the parent class of all objects in the language [source] . This leads to inefficiencies due to how memory is handled.\n\nThe fact that numpy's core is written in C means that an array points directly to all its values located in a contiguous block of memory. This allows for much faster execution time as the cache memory can leverage the spatial locality of the data. You can read more about it here."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: fab9d7f8-f168-4421-8089-939ae3809433
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Because pandas stores strings as objects, it has to fall back on slow Python arrays. In contrast to numpy, a Python `list` has a pointer to a memory-contiguous buffer of **pointers**, which point to objects stored in memory, but which also reference data stored in other locations. Hence, accessing this data during runtime will be quite slow in comparison. Source"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: f9480162-419a-46f0-8cfd-51b5f30a16b8
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Because pandas stores strings as an array of objects, it has to fall back on slow Python arrays. In contrast to numpy, a Python list has a pointer to a memory-contiguous buffer of pointers, which point to objects stored in memory, but which also reference data stored in other locations. Hence, accessing this data during runtime will be quite slow in comparison.\n\nA Python list and how it handles data. Source\n\nIn a nutshell, pandas is constrained in part by the ecosystem. But how can we make the best of the tools we have right here, right now? Well, it\u2019s a different story for numerical and string data.\n\nNumerical types"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d6919e96-d15a-4c24-8378-12333283c6b5
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Numerical types\n\nRegarding ints and floats, downcasting is the key to saving memory. Pandas supports 8, 16, 32, and 64-bit signed and unsigned integers and 16, 32, and 64-bit floats. By default, it opts to use 64-bit variants for both types.\n\nThe trick is to choose the smallest type that can comfortably hold your data. You should consider both current and new valid data. You don't want to limit yourself to unsigned ints in your pipeline if signed ints are a real possibility.\n\nIn practice, and for your current dataset, you can easily do this with pandas' own pd.to_numeric like so:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 7804e345-a032-4437-960d-40ea01bc94d9
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "In practice, and for your current dataset, you can easily do this with pandas' own pd.to_numeric like so:\n\nJust like that, we reduced df 's size by 1/3. But your mileage will vary based on your data. As a side note: pd.to_numeric only downcasts floats to float32 , it doesn\u2019t use float16 even though it is compatible with pandas.\n\nRegarding data types, we found that bool should be favored if you have no nulls. It occupies the same amount of memory as an uint8 , but makes the column\u2019s content much clearer.\n\n\u201cWhat if I have NaN values?\u201d We hear you. In principle, you can cast your ints and bools to float32 and call it a day. You\u2019ll take a memory hit, but NaN s will be handled natively."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: ec0b2219-4e22-49e2-9073-df87e796ad7b
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Somewhat recently, pandas has added its own Nullable integer and Nullable boolean data types. They can be more memory efficient than float32 but with the caveat that neither .values nor .to_numpy() return correct numpy arrays with np.nan (because it is itself a float, not an int). Internally, these arrays hold a _data array with the appropriate numpy dtype and a _mask , where True means that the value in _data is missing. This can be an issue if you\u2019re trying to use numpy. The latter and their experimental nature are why we can't fully recommend their use for now.\n\nThis is also why we don\u2019t recommend using .convert_dtypes just yet because it uses nullable types.\n\nString types"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d0d1ad44-114d-4a66-b9be-581ef5c82f2f
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "This is also why we don\u2019t recommend using .convert_dtypes just yet because it uses nullable types.\n\nString types\n\nWith numbers out of the way, let's dive into strings. Strings are used everywhere, but they're the bane of our existence when manipulating data. As mentioned before, they're stored with the inefficient object type, and that's why they use lots of space and are slower than more specialized types."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: e3183146-c157-49f8-b1c8-2298720f0d4c
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "But there\u2019s a glimmer of hope with StringDType. Like the other nullable types we mentioned, StringDType is still experimental, so we don't recommend using it in production code for now. Looking ahead, it promises correct handling of missing values and performance improvements over object ."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: a60e147a-f7e9-4e02-a6b2-f7e30a3f2ca3
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "If your column has low cardinality (many repeated values), the smartest move will be to use pd.Category . nternally, a categorical column holds a mapping from the actual column value (i.e., the contents of your column before conversion) to some arbitrary integer. Then, the column is saved as an int type, exploiting the advantages of contiguous memory and efficient storage. Furthermore, Run Length Encoding is used to reduce the storage size even further (if an integer is repeated, it only saves the integer and the number of instances rather than all the integers).\n\nThis results in excellent compression and processing speed, as seen in the results subsection.\n\nResults"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: fcfedcba-7388-490f-b33a-f4558e3fa7c6
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "This results in excellent compression and processing speed, as seen in the results subsection.\n\nResults\n\nNow that you have a better understanding of the different dtypes and their efficiency, let's look at some of the tests we ran and their results:\n\nWe tried using pandas' convert_dtypes in both df1 and df2 . We ran a custom convert_dtypes that doesn\u2019t use pandas' nullable types. We converted applicable string columns into categoricals. We downcasted numeric types into their smaller types.\n\nOf course, these results will vary according to your data, but we wanted to highlight the benefits of choosing the correct data type. Here are the results in table form:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 8585e8d8-0466-42cc-ab93-9e2d83e8e6fd
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Description df1 df1 df2 df2 Description Size [GB] Ratio Size [GB] Ratio Size on RAM (unoptimized) 24.8 1 12.0 1 pandas' convert_dtypes 24.9 1.004 12.5 1.04 Our convert_dtypes 24.7 0.996 12.6 1.05 Convert to Category 13.8 0.556 2.3 0.19 Downcasting 12.5 0.504 1.7 0.14\n\nWe managed to reduce the dataset size to about 1/2 and 1/8, respectively, as soon as pandas finished loading it.\n\nThis is important because we can now process the data in more ways than before simply because we have more free RAM. And our entire system will likely be snappier as well.\n\nAlso note that pandas' convert_dtypes enlarges the dataset in both cases."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: ea55afff-fdcb-4af6-8dd3-6d190eb86c0b
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Also note that pandas' convert_dtypes enlarges the dataset in both cases.\n\nAgain, you might be asking, why bother with convert_dtypes if it barely saves any memory?\u201d Or, for that matter, how come these people always know the questions I\u2019m about to ask? Well, our job is to predict the future, so we've gotten quite good at it. And we're glad you asked because the speed improvements completely dwarfed the memory reduction benefit. Try running the following code to compare your times:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: cc1a24b5-2555-41aa-a2eb-c803c03e4d6f
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "With our data, we saw speedups of between 12x to 50x. For the mean, the speedups were 12x to 25x. For the standard deviation, they were 20x to 50x. For value_counts , they were about 15x. That\u2019s not bad for such a small amount of work!\n\nCategoricals, in particular, reap the benefits twofold. Not only is the memory usage reduction reason enough to use them, but they make operations (like groupby or str.lower() ) much faster. Notably, .str.lower() was about 22.5x faster on categorical columns than on regular object columns (or StringDType columns for that matter)."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: c1b3ca94-66ea-49d3-86c9-054c72763c95
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "groupby s are another can of worms. In our testing, when grouping with 1 column, categoricals were about 1.5x faster. However, when grouping with multiple columns, where at least one of them is categorical, pandas defaults to creating an index that is the cartesian product between all categories in said columns. You can imagine how this explodes in size and complexity quickly."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 8f5e2817-82a6-465c-8937-de4ad1fe1e7e
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "But don\u2019t despair. There\u2019s a solution! Okay, you can despair just a little because the solution isn\u2019t a catch-all. Some categories are mutually exclusive; therefore, their combination, which would appear in the cartesian product, would never be observed in the data. You can use the observed=True kwarg of groupby to instruct pandas to execute the operation in this way.\n\nIIn our tests, we measured a 2.5x speedup using observed=True , while we got a slowdown of about 2x when we didn\u2019t. If you notice this doesn\u2019t work well for you, converting to object , performing the groupby , and converting back to categorical could be the way to go.\n\nTip 5: Long-term storage"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: a6ed881b-df5b-4966-ba25-9f272f9740f5
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Tip 5: Long-term storage\n\nCSV, parquet, pickle, JSON, XML, HDF5, Feather, ORC, STATA, SAS, SPSS, or, dare I say it, Excel (yuck!) are some of the formats that pandas supports. Argghhh! There are too many options! How are we supposed to choose?\n\nWe might start, as any responsible engineer would, by reviewing the pros and cons of each format. We\u2019d then test a few that stand out in the real world and come to some conclusion about which format we\u2019d favor and when. Life is tough, though, and with looming deadlines imposing time constraints, the selection process looks more like this:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: e631b042-e997-47f7-af60-5af765511a3d
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Fear not! We've done our homework - and we promise that our method is more scientific than Homer's. We recommend you take a good look at parquet. You don't need to constantly be making these decisions anymore.\n\nParquet was created to help developers reap the benefits of a \"compressed, efficient columnar data representation.\" It supports different encodings and can compress on a per-column basis. That fits the ML practitioner's use case like a glove!\n\nIt sounds awesome; sign me up! Right? Go on, try to do a simple df.to_parquet . I\u2019ll wait\u2026"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 38d304da-4179-41e0-90bb-303b3115aa7b
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "It sounds awesome; sign me up! Right? Go on, try to do a simple df.to_parquet . I\u2019ll wait\u2026\n\nAlready back? Did you get an ImportError ? Parquet is technically a format, which is more like a spec than an implementation. So, to use it, you need to install another library that implements the format. Currently, pandas supports pyarrow and fastparquet .\n\nBut you promised no more decisions! Well, we believe pandas\u2019 use of pyarrow by default is the right choice. We found it to be more reliable than fastparquet . For example, when trying to save and then immediately read df2 , fastparquet complained about an overflow."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 57742642-de2e-4d05-8655-ad19c6dddee7
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Below is a table with read/write durations and the resulting sizes of the datasets saved in different ways (with pyarrow ). It boils down to this: use parquet and change the compression method to fit your needs. Uncompressed or snappy parquet ought to be your choice during development, resorting to gzip compression for long-term archival."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 7aa34759-aef6-4e6d-b662-74050ceba802
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Description df1 df1 df1 df2 df2 df2 Description write [s] read [s] size [GB] write [s] read [s] size [GB] pickle 17.7 9.9 6.5 0.9 0.6 1.3 pickle gzip 532.2 40.1 3.8 99.8 3.0 0.2 pickle lzma 2670.5 158.0 3.2 187.9 7.7 0.1 pickle bz2 455.3 218.1 3.5 127.9 19.5 0.2 parquet 22.3 16.0 6.4 3.6 1.3 0.4 parquet gzip 299.5 24.7 4.0 21.5 1.5 0.2 parquet snappy 25.5 15.5 5.9 4.0 1.3 0.3"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 9795c46f-3b89-4df2-b02f-32ea2eb1ccb9
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "CSV is probably in the top 5 data distribution formats, and that's in part thanks to its ease of use. With a simple df.to_csv , you can create a file that anyone (and their dogs) can open in the program of their choice. But, and that's a big but, CSV is severely limited in terms of volume, size, and speed. When you hit these limitations, it's time to start exploring parquet.\n\nThe latest pickle version is fast but has substantial security and long-term storage issues. The words \"execute arbitrary code during unpickling\" should send shivers down your spine, and those are the exact words in the massive, red warning in pickle's official documentation."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 19dda546-3d18-4ebf-a1a0-b93b88bc80b4
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "You might think, \"well, that's a non-issue; I'll only unpickle stuff I've pickled,\" but that won't save you either. When unpickling, the object's definition at that time is used to load it from storage. That means that if your objects change, your pickles could break. In the example below, which raises an AttributeError , we show how removing an attribute from class C after saving the pickle results in the loaded object losing the attribute despite it being there when it was saved.\n\nNow imagine pandas changes something in the DataFrame class. You might run into issues when unpickling older DataFrames. Favor (uncompressed) parquets over pickles to avoid this."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: f7dc62ae-017f-4ce0-9425-b0191781e09a
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "We recently came across a tool called Lance that aims to simplify machine learning workflows. It supports a columnar data format called Lance Format, which claims to be 50 to 100 times faster than other popular options such as Parquet, Iceberg, and Delta. Although we have yet to thoroughly test it, the potential speed increase is intriguing, and we encourage you to give it a try.\n\nConclusion\n\nPhew! That was a loaded blog post, full of insights and valuable information. We hope you apply some of these tips in the real world and supercharge your codebase to be faster and more powerful.\n\nHere are some of the key concepts to keep in mind:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 87689a4b-3e78-4d8b-92c0-533e16167439
encoding: FLOAT32
metadata:
{"title": "Top 5 tips to make your pandas code absurdly fast", "link": "https://tryolabs.com/blog/2023/02/08/top-5-tips-to-make-your-pandas-code-absurdly-fast", "language": "en", "description": "Learn how to optimize your pandas code for large datasets with these top five tips. From vectorizing operations to embracing NumPy, our expert advice will help you get the most out of your pandas workflow.", "chroma:document": "Here are some of the key concepts to keep in mind:\n\nPython\u2019s dynamic nature makes it slower than compiled languages. This issue is exacerbated in scientific computing because we run simple operations millions of times. Don\u2019t loop over your data. Instead, vectorize your operations (remember Hector Vector). You can use numpy directly by calling .to_numpy() on the Dataframe, which can be even faster. Choose the smallest possible numerical dtypes, use categoricals, and prefer float32 over the newish nullable dtypes (for now). Use parquet to store your data. Use snappy compression (or none at all) during development, resorting to gzip for long-term archival."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 0b0b6919-3c48-4b0f-acd6-3ff4e4513e82
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Welcome to the 9th and not-yet-quite-a-decade edition of our yearly Top Python Libraries list! As we journey through the constantly evolving landscape of Python development, it's time again to spotlight the standout libraries and tools that have caught our attention this year. Since our first edition in 2015, we\u2019ve dedicated ourselves to exploring the depths of the Python ecosystem, and this year is no exception. In fact, it's special \u2014 2023 marks the boom of Generative AI and Large Language Models (LLMs), a trend that has significantly influenced our picks.\n\n2021, 2020, 2019, 2018, 2017, 2016, 2015. See the other posts in this series: 2022"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d284d853-bec1-4b4a-a477-21dbc725e169
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "2021, 2020, 2019, 2018, 2017, 2016, 2015. See the other posts in this series: 2022\n\nOur criteria remain consistent: we're on the lookout for libraries and tools that have been launched or risen to prominence over the past year. To qualify for our top picks, they need to be well-maintained, innovative, and, of course, cool enough to spark your interest. This year, you'll notice an emphasis on libraries that align with the LLM wave, reflecting how these technologies are reshaping the Python (or should we say programming?) landscape."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6a6c254d-8f33-4d17-bebd-f60c3ba5ce09
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "But fear not, diversity in our selection is still key! Although getting increasingly more difficult every year, we tried to ensure that our full list is not just for AI enthusiasts but also contains gems that can be valuable for a wide range of Python developers, including some niche applications.\n\nOur aim with this post is also to ignite conversations about the remarkable libraries and tools that we might have overlooked.\n\nSo, with eager anticipation, let's dive into the world of Python in 2023!\n\nThe 10 main picks\n\n1. LiteLLM \u2014 call any LLM using OpenAI format, and more"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: dd46d389-31bb-4fbe-a09e-42a758a0aed9
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "The 10 main picks\n\n1. LiteLLM \u2014 call any LLM using OpenAI format, and more\n\nImagine a world where you're not tethered to a specific model provider, where switching between LLMs is as easy as changing one or zero lines of code, contrary to rewriting your entire stack using tools like LangChain. LiteLLM makes this a reality.\n\nLiteLLM's intuitive and non-intrusive design allows seamless integration with various models, providing a unified format for inputs and outputs, regardless of the underlying LLM. This flexibility is a game-changer for developers who need to adapt quickly to the rapidly changing landscape of large language models."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 94374555-6925-472d-9067-146a62cc4b49
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "But LiteLLM isn't just about flexibility; it's about efficiency and control. Features like streaming responses and mapping exceptions to OpenAI\u2019s format demonstrate its robustness. The ability to track and manage costs across different LLMs is particularly noteworthy, ensuring that your AI explorations don't lead to unexpected bills. This level of financial control is vital in a domain where API calls can quickly accumulate costs.\n\nFurthermore, LiteLLM comes with a proxy API server, enhancing its versatility in integrating on codebases without changing a single line of code \u2728."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 85a0279c-d4cb-4bb6-9de9-93086b90bd4a
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Another differentiating feature is the built-in load balancing and rate limiting features. These capabilities ensure optimal performance and resource utilization, especially when dealing with high volumes of requests across multiple inference calls. Load balancing efficiently distributes incoming API calls, preventing any single model provider from becoming a bottleneck. Meanwhile, the rate limiting function plays a crucial role in maintaining stable and controlled access to the models, safeguarding against excessive usage and potential overloads (forget HTTP 429)."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6c13ae5d-ed2c-4cc5-8ecd-975fa9883be4
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Of course, the devil is in the details: prompting will work differently for different LLM providers, so you will need to do some work if you truly want to utilize different providers. But it\u2019s good to have a layer that you can rely on to make the switch as easy as possible.\n\nCheers and congrats to the LiteLLM authors. Whether you're a seasoned AI developer or just starting out playing with Generative AI, LiteLLM is a library that belongs in your toolkit!\n\n2. PyApp \u2014 deploy self contained Python applications anywhere"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: ab3acff4-8d3c-4eff-9e78-367599c33ce4
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "2. PyApp \u2014 deploy self contained Python applications anywhere\n\nPython's surge in popularity over the years is no secret. Its blend of simplicity and effectiveness, coupled with a vast and friendly community, has made it one of the most widely used programming languages globally. Yet, distributing Python applications remains a long-standing challenge. Unlike other languages that allow distribution of standalone compiled binaries, Python has struggled in this area, especially when targeting non-technical users. Traditionally, Python applications are shared via Docker container images, a method not always accessible to all users."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6bcf4570-b580-4427-887a-e5f114aec86a
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Enter PyApp: a utility library that simplifies the distribution and installation of Python applications. It achieves this by embedding Python within a self-installing package that is compatible across all operating systems. But it doesn't stop there! PyApp also supports self-updating capabilities, and can be customized to accommodate a variety of use cases."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: c7a2c671-e251-46d6-9201-7caae1680a6d
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "With PyApp, specifying dependencies for pip installation via requirements.txt files is a breeze, and additional artifacts can be seamlessly embedded into the package using .in files. The library handles the installation process, creating a dedicated directory for Python and the app's required dependencies. On subsequent runs, PyApp checks for the existence of this installation directory, optimizing command-line interface responsiveness.\n\nIt differentiates from other tools you might know like PyInstaller in the fact that PyApp doesn\u2019t do dependency discovery but rather relies on an explicit declaration of dependencies which seems to solve some issues users have been facing."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: b1c9c994-f196-4fb0-8af0-15738de60b9c
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "In a nutshell, PyApp presents a straightforward yet potent solution for delivering Python applications to your users. It streamlines distribution and installation procedures, eliminating the hurdles typically associated with these processes. As we look forward to 2024, PyApp is undoubtedly a tool worth adding to your Python arsenal, making the distribution of Python apps as easy as pie!\n\n3. Taipy \u2014 build UIs for data apps, even in production"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 5d72077a-2b5a-42ff-a3ce-7a484543766a
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "3. Taipy \u2014 build UIs for data apps, even in production\n\nIn the wise words of fashion designer Giorgio Armani, \"Elegance is not about being noticed, it's about being remembered\". This philosophy is the guiding star of Taipy, a Python library that empowers data scientists to weave engaging narratives from their data."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: c9971d46-dff0-49b2-a91a-0196d98f0c49
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "In the realm of data science, the art of storytelling can be as crucial as the underlying models. And having the right tool can make all the difference. That is Taipy\u2019s game. Taipy is a low-code Python library that enables data scientists to build interactive Web UIs for their machine learning products, without the need to master web stack tools. It's all about allowing data scientists to focus on what they do best - extracting insights from data."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: cb04dd2c-18e3-4179-8cc7-207fefc525a5
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Taipy's magic lies in its ability to bind variables and expressions to the state of visual components in the UI. This binding happens in-place when declaring the visual components, drastically reducing boilerplate code needed to develop your app. And speaking of visual components, Taipy offers a rich array of options, including plots, sliders, and image grids, to help you build next-gen UIs."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 4a2687f8-8603-4e91-839c-4fd4047f752a
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "But wait, didn\u2019t we have Streamlit (our pick #7 of 2019) for that? Well, that\u2019s where Taipy shines. Unlike Streamlit, which reruns code for the entire page upon detecting an input change, Taipy employs callbacks to update only the components affected by the change (much like shiny does for R for those coming from our brother language) enhancing responsiveness and accelerating development."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 1e0b9c92-b5b9-4149-8684-f4ce66ace684
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "But the innovation doesn't stop there. Taipy also provides tools for visually composing machine learning pipelines, speeding up development while offering an easy way to explain at a high level the steps your model takes. Importantly, data nodes, the inputs to these visual pipelines, can be dynamically changed, enabling interactive exploration and what-if scenarios.\n\nWhether you choose to self-host your Taipy applications or use Taipy Cloud for serverless deployment, you're in for a seamless experience. Taipy helps you unveil the stories hidden in your data and create memorable, interactive machine learning workflows. Check it out today!"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d3b73d9a-f6d4-4480-b8a6-3182aa19173a
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Here\u2019s their 2023 PyData Seattle presentation and here\u2019s their launch post on HackerNews; be sure to check the discussion for some interesting information nuggets on the lib.\n\n4. MLX \u2014 machine learning on Apple silicon with NumPy-like API\n\nIn 2021 Apple introduced it\u2019s M1 architecture, now further refined into the current M3, a revolutionary piece of hardware that brought along performance and efficiency improvements, as well as a unified memory buffer that can be accessed by the CPU, GPU and NPU (Apple\u2019s Neural Engine) without the need to copy information between processing units."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 69671d72-0b32-4596-a016-65304777fa59
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "In 2023, the spotlight is on Apple's MLX, a groundbreaking array framework specifically designed for machine learning on Apple silicon. Brought to you by Apple's own machine learning research team, MLX is setting new standards for the development and deployment of machine learning models on Mac hardware.\n\nMLX is designed to retain familiarity: its Python API is a close cousin of NumPy's, and it also offers a fully-featured C++ API that mirrors its Python counterpart. Higher-level packages like mlx.nn and mlx.optimizers follow the footsteps of PyTorch, making the construction of complex models a breeze."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 1195e83d-a199-4201-8be5-7c63b4e09688
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "But MLX is more than just a familiar face. It introduces innovative features like composable function transformations, enabling automatic differentiation, automatic vectorization, and computation graph optimization. It also employs lazy computation, materializing arrays only when needed, and dynamic graph construction, making debugging more intuitive and avoiding slow compilations due to changes in function argument shapes."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: e1b4c97b-fbf3-4554-b21b-07f17e00e040
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "MLX's multi-device capability allows operations to run on any supported processing unit, be it CPU, GPU or NPU. And here's the game-changer: MLX's unified memory model. Unlike other frameworks, MLX arrays exist in shared memory, enabling operations on any supported device without the need for time-expensive data transfers.\n\nThis user-friendly yet efficient framework is designed by machine learning researchers for machine learning researchers. Inspired by the likes of NumPy, PyTorch, Jax, and ArrayFire, MLX aims to make it easy for researchers to extend and improve the framework, fostering rapid exploration of new ideas."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: a1ca2c39-b6fb-4207-b53a-c048accfba97
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "As we move towards 2024, MLX is definitely a toolkit to keep an eye on. Whether you're an Apple aficionado, a machine learning enthusiast, or both, MLX is poised to revolutionize your machine learning journey and utilize the capabilities of Apple silicon to the fullest.\n\n5. Unstructured \u2014 the ultimate toolkit for text preprocessing\n\nIn the vast landscape of AI, a model is only as good as the data it feeds on. But what happens when your data is a chaotic jumble of raw documents, scattered across various formats? Enter Unstructured, a comprehensive solution for all your text preprocessing needs."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 50a8137e-dd03-49ce-8a1e-8a6fdcdbc841
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Unstructured is the master key to unlock the potential of your raw data, capable of digesting anything thrown at it \u2014 PDFs, HTML files, Word documents, even images. It's like a gourmet chef for your data, slicing and dicing it into palatable pieces for your AI models.\n\nUnstructured offers a smorgasbord of functions, neatly categorized into six types:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 129e9c27-195e-4690-92e7-227a33c17fe7
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Partitioning Functions that transform raw documents into structured and standardized formats. Cleaning Functions that scrub away unwanted text, leaving behind only the relevant nuggets. Staging Functions that format your data, readying it for ML tasks like inference or labeling. Extraction Functions that mine for valuable information within documents, like emails, IP addresses, or pattern-matched texts. Chunking Functions that break down documents into bite-sized sections, perfect for similarity searches and RAG. Embedding Functions that convert preprocessed text into vectors, offering easy-to-use interfaces."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 1b6f53a3-d0bd-4dbf-8b44-2f1f440fa671
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "But that's not all. Unstructured also comes equipped with Connectors, a set of tools for seamless integration with various storage platforms. From GCS and S3 to Google Drive, One Drive, Notion, and many more. These Connectors work both ways, allowing you to read and write information to and from a plethora of sources.\n\nUnstructured is your go-to Python library for effortless text extraction from documents. It simplifies text cleansing, handling everything from bullet removal to emoji management and language translation. Its extractor functions parse cleansed text, retrieving valuable information. You can use built-in extractors or custom templates for structured text."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 1b6ca150-8ca7-48fb-a67b-8baa0c69488e
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "The library's staging functions save preprocessed documents in diverse formats, compatible with various tools like Label Studio and Prodigy. It also features a chunker that groups texts by their title building cohesive, related sections and embedding functions for easy text vectorizing using popular LLM models.\n\nWith Unstructured in your toolkit, your journey into the world of Large Language Models is set to be a rewarding one. Get ready to unlock the full potential of your data in 2024!\n\n6. ZenML and AutoMLOps \u2014 portable, production-ready MLOps pipelines\n\nAlright, we're bending the rules a bit here. Two libraries, one entry. But trust us, it\u2019ll be worth it."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 0cb4aec3-7763-490c-ab61-a9bbc26e7456
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Alright, we're bending the rules a bit here. Two libraries, one entry. But trust us, it\u2019ll be worth it.\n\nMLOps is a crucial link between machine learning model development and operations. It enables a smooth transition from experimental to production-ready models. Without MLOps, organizations face navigating a minefield of inefficiencies, wasted resources, and unreliable predictions. It ensures model reproducibility, scalability, and monitoring, transforming brilliant ideas from mere theories into valuable solutions. Thus, effective MLOps is the backbone of successful machine learning initiatives."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: f09a2f70-1347-438d-aff3-695529493cb9
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "However, setting up an effective MLOps pipeline can often feel like a Herculean task. The sheer range of services and skills required can make the process daunting and time-consuming. Furthermore, custom developments often become so intertwined with application logic that maintaining application code becomes a challenging endeavor.\n\nThese picks are designed to alleviate these issues and put MLOps best practices front and center of your development."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: f8185af3-2489-48d1-a7dc-e20b8c6f8c10
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "These picks are designed to alleviate these issues and put MLOps best practices front and center of your development.\n\nZenML provides a fast-lane for transitioning from experimental code to a structured, production-ready pipeline with minimal changes. This vendor-agnostic framework excels at decoupling application code from infrastructure tooling, allowing you to specify different service providers for each component of your infrastructure."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: bc1a1b6a-c622-42a6-b280-c5c8cff8077f
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "With ZenML, your pipeline\u2019s steps are annotated and linked together to form the actual pipeline. Then, you configure a Zen Stack, specifying the necessary tooling and infrastructure. This stack consists of two main components: the orchestrator and the artifact store. The orchestrator, which could be a cloud execution service or a simple Python interpreter, runs your pipeline's code. The artifact store is where your data will reside. And the best part? ZenML supports numerous integrations, offering additional functionality like feature stores, data validation, and experiment trackers."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d30b5c1a-4b26-4dd5-92d4-aafa13561fbd
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "In the other corner, we have Google's AutoMLOps, which shares ZenML's approach of annotating model development code as components and pipelines. However, AutoMLOps takes it a step further by automatically generating the necessary code to containerize your components, provision them to Google\u2019s cloud, and deploy those containers to the allocated resources. All of this follows an automated, reproducible CI/CD process that helps accelerate your path towards MLOps maturity level 2."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 397056e7-e5cd-420c-ab83-9d3d4fd85fc7
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "The deployed pipelines can then make use of Google\u2019s extended MLOps suite of functionalities, ranging from data processing and storage, to experimentation, training and offline evaluation to online serving with configurable auto-scaling capabilities and testing as well as monitoring for drift both in your input data and in your models\u2019 predictions. This enables use of advanced capabilities like continuous training and deployment, enhancing your applications\u2019 accuracy and catching errors before they can have a real impact on the end-user or business."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: ca1c75d8-800e-49cc-a472-bd86dc1400e7
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Uniquely tailored to leverage Google Cloud Services and Vertex AI, AutoMLOps is your magic carpet ride through the cloud. While it's true that this tool is unabashedly Google-centric, it's a small concession for the vast benefits you'll reap. If you're open to navigating the Google Cloud, prepare to have a significant portion of your journey streamlined, making your cloud-based endeavors more efficient and enjoyable than ever.\n\nWith these two powerhouses in your toolkit, you're well-equipped to navigate the complexities of MLOps, ensuring your machine learning initiatives are not only successful but also efficient and manageable."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 10cfffd6-da33-476b-bb2d-5bb12a07b291
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Say goodbye to the days of wrestling with infrastructure and application code, and say hello to the era of streamlined, effective MLOps.\n\nHave you crossed paths with Whisper, the trailblazing speech recognition model from OpenAI? It's like a multilingual maestro, adept at a concert of tasks such as speech transcription, timestamping, speech-to-text translation and voice detection \u2014 all in various languages."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 991fe6d7-ccba-41f5-b45b-e92faa930166
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Now, let me introduce you to its turbo-charged cousin, WhisperX. It's like Whisper after a double shot of espresso, boasting more accurate timestamping, multiple speaker detection, and a reduction in hallucinations (not the psychedelic kind, but the kind where it starts pouring words when no-one is speaking) by enhancing the voice activity detection. The best part? It does all this while being faster and less memory-hungry."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 977adb9b-c6af-4485-bc58-6b0f4fade0a8
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "WhisperX has a secret weapon called voice activity detection preprocessing. It's like a bouncer that only lets in audio segments that contain voice activity, and it's much cheaper than letting everyone in for a full automatic speech recognition party. This approach also ensures that the audio is sliced into neat chunks, avoiding any awkward cuts mid-speech and boosting the reliability of whole word detection. Plus, the timestamps are tidied up to match these boundaries, making them even more accurate. The final touch? A phoneme classifier model that aligns word boundaries based on a fixed, per-language dictionary. It's like having a grammar stickler on your team."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 60c86921-c169-44c0-a97e-42c025ba1ea8
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Curious to see it in action? Check out an English sample of the results on their GitHub repository.\n\nThe beauty of WhisperX is that the chunks can be processed in parallel, like a well-oiled assembly line, which enables batched inference. And thanks to a switch to the Faster Whisper backend, the RAM usage for the whisper-large model has been trimmed down to less than 8GB. This means it can now play nice with a wider range of graphics cards while also boasting a whopping 70x speedup on the same model. Like putting your speech recognition on a high-speed treadmill.\n\nFor the curious minds who love to dive into the nitty-gritty, the authors have generously shared their pre-print paper."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 342fbca0-baf2-446d-a525-9fe6300fbcbb
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "For the curious minds who love to dive into the nitty-gritty, the authors have generously shared their pre-print paper.\n\n8. AutoGen \u2014 LLM conversational collaborative suite\n\nAutoGen: Agent Collaboration with Human Intervention. Source: AutoGen Multi-Agent Conversation Framework\n\nEver dreamed of a personal team of software engineers ready to bring your app ideas to life? Meet Microsoft\u2019s AutoGen, your magic wand that conjures up conversational agents working in harmony to achieve a shared goal. It's like the Avengers, but for software development."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 332ca6c6-712d-439b-95aa-4d2c4867b12a
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "AutoGen's simplest setup stars two key players: the AssistantAgent and UserProxyAgent . The AssistantAgent is your AI sidekick, designed to execute tasks independently. Meanwhile, the UserProxyAgent is your digital stand-in, allowing you to step in when human intervention is needed but also being able to act on your behalf. It's like having your cake and eating it too!\n\nThe AssistantAgent performs tasks, while the UserProxyAgent provides feedback to the AssistantAgent on whether the task is succesfully completed. Once the task is set, these agents spring into action, working autonomously until they either require human oversight or they've successfully completed the task."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 465e30d1-d53d-479c-b732-138e0c42cdd5
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "This framework shines in use cases like code generation. The AssistantAgent crafts the code, while the UserProxyAgent tests it. If bugs appear, the UserProxyAgent provides feedback, and the cycle repeats. It's like owning a personal code factory.\n\nThe beauty of AutoGen lies in its scalability. You can introduce additional agents to fulfill various roles, akin to building your software development team. Need a tester agent or a product manager? AutoGen has got you covered. For inspiration, check out ChatDev, a project that built an entire software team using a similar concept."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 368b38c8-dfe1-4eb4-a3c5-ed992037cf85
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "AutoGen also boasts several Enhanced Inference features that empower your workflow. It unifies the API for several models, enabling you to link multiple LLMs behind a single agent: if one model fails the next in line takes the mantle, ensuring robustness. Furthermore, it provides a prompt results cache for faster, cost-effective inference, chat logging for debugging and prompt templating so that common instructions (such as explain your reasoning step by step) can be reused across multiple tasks."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 57b33724-5326-4787-8d8c-55752ba969eb
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "But that's not all. Do you want to enhance your agents with RAG? Just add Retrieve to their class names and provide a docs path et voila! Your agents can now tap into custom databases providing context-specific information. Want an agent that learns on the go? Try explaining via prompts some stuff to the TeachableAgent and it will store the information you provide in a MemGPT-style embeddings DB, allowing it to recall and reason over this context in future interactions. An extremely powerful usage for this is having your agents reason about what worked in their interactions and produce a summary recipe of the steps that did work, then teach the TeachableAgent that recipe so it knows how to"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6145a999-c727-4476-88c2-976c8f555cad
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "worked in their interactions and produce a summary recipe of the steps that did work, then teach the TeachableAgent that recipe so it knows how to do it next time."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: cbb4e6b2-9c77-4a7e-b517-c568b6333231
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "To cap it all off, AutoGen offers robust security measures. The AutoGen team strongly advises running your agents' code inside a Docker container. You know, just in case you stumble upon the agent combo that unlocks AGI. \ud83d\ude09\ud83d\ude09. With AutoGen, the sky's indeed the limit.\n\n9. Guardrails \u2014 babysit LLMs so they behave as intended\n\nHave you ever had a fantastic prompt, carefully crafted through a meticulous prompt-engineering process only to find your Large Language Model breaking its chains and responding with a non-structured answer? Or an LLM not following your instructions and mentioning your competitor?"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 32650efa-31bd-4b6f-b221-b639cd261537
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "The flexibility of LLMs provides an immensely vast range of applications but also makes its integration into a greater pipeline challenging. Enter Guardrails: a library designed to specify structure and type, as well as validating and correcting the outputs of large language models.\n\nGuardrails works by defining a .rail specification, a flavor of XML, that\u2019s intended to be human readable and allows for defining the structure of the requested output, validators to ensure the values meet a given quality criteria, and corrective actions such as re-asking the LLM or filtering out the invalid output if the validations are not met."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 36e795bb-306d-4ee1-b631-a62e8e50bfc4
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Here\u2019s an example that specifies the quality criteria (generated length, URL reachability) in the format fields of the RAIL spec, re-asks for explanation and filters the follow_up_url if not valid."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: ebf57cae-52a2-4ca9-97c3-12fba547ec84
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Guards such as these can be initialized from a RailSpec as well as from PyDantic type definitions, or even strings for less complex use cases. The structure can then be used to define the fields that the LLM must produce as well as their types, including nested objects defined within your logic. Several validators are provided out of the box ranging from simple string min/max length or regex validations to syntactic bug-free Python or SQL code, profanity-free text and many [more] (https://docs.guardrailsai.com/api_reference/validators/).\n\nWith Guardrails in your stack, your LLM applications are bound to be more reliable and robust, making it easier to integrate them into your flows."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 396e49f8-90ef-45a1-ad63-3819fbd88ed8
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "With Guardrails in your stack, your LLM applications are bound to be more reliable and robust, making it easier to integrate them into your flows.\n\n10. Temporian \u2014 the \u201cPandas\u201d built for preprocessing temporal data\n\nTime is of the essence. Time is money. Time is running out\u2026 You\u2019ve heard it before right? Temporal data is ubiquitous in most human activity, that's why many popular libraries allow representing and manipulating time-series data: a series of values with equal time intervals between them."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 5ce77185-6494-4229-8a38-1b006b3f9287
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "However, the classical time series approach falls short for representing many kinds of real-world temporal data that is not equally spaced, such as user clicks on a website or sales happening in several of a retailer\u2019s stores.\n\nThe go-to approach in this scenario has always been to use a general data preprocessing tool, such as pandas, and cram data into a tabular format by resampling and aggregating it \u2014 losing both flexibility and granularity in the whole feature engineering process."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 12b395bb-ef6a-4a78-8554-5e1c44e03990
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "This is why Tryolabs has partnered with Google to develop a state-of-the-art library for temporal data manipulation. Temporian allows for safe, simple and efficient preprocessing and feature engineering of temporal data through a Python API backed by a C++ implementation for blazing-fast speeds. Temporian supports multivariate time-series, multivariate time-sequences, event logs, and cross-source event streams."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 333e93ef-e8e8-405b-a868-cbead33277c2
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "At the core of Temporian is the EventSet , which contains timestamped records called Events that hold feature values. EventSets can contain multivariate time series with uniform or non-uniform timestamps, as well as independent series belonging to different entities. Handy operators like lag, resample, and window functions help fill in gaps and align timestamps for arithmetic operations.\n\nTo get a sense of the API, here\u2019s a simple Temporian snippet that computes the daily revenue for each store in a sales dataset, sampled at 10pm every workday:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 2a01ea3a-43fe-4196-9d16-e301a36d039d
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Temporian proposes a new paradigm through which to work with temporal data. And being tailor-made for it, plus its core computations running as highly-optimized C++ code, enables it to make commonplace temporal operations safer, easier to write, and much, much faster to execute \u2014 with benchmarks showing speedups of upwards of 100x against equivalent pandas code.\n\nIf you are curious, check out the Tutorials in the documentation, which will walk you through real examples using the library. And of course, all feedback is more than welcome :)\n\nRunner-ups\n\nThe following libraries almost made it to our top picks and we feel they deserve special recognition:"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: e00e6a39-2089-4a53-88aa-73c1f83b5c7d
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Runner-ups\n\nThe following libraries almost made it to our top picks and we feel they deserve special recognition:\n\nDataGradients \u2014 say hello to your personal assistant for computer vision dataset analysis . DataGradients, a brainchild of Deci AI, is an open-source Python library that comes packed with solutions for common tasks in vision dataset management . Whether it's identifying corrupt images, wrong labels, biases, extracting summaries for a deeper understanding of available data, or providing guidance for hyperparameter selection, DataGradients has got you covered."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6d30eee8-a58a-48d9-acbc-9a14caa03d32
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 say hello to your personal assistant for . DataGradients, a brainchild of Deci AI, is an open-source Python library that comes packed with solutions for common tasks in . Whether it's identifying corrupt images, wrong labels, biases, extracting summaries for a deeper understanding of available data, or providing guidance for hyperparameter selection, DataGradients has got you covered. functime \u2014 a comprehensive library for time-series forecasting and feature extraction on large panel datasets. Offering unique preprocessing options and cross-validation splitters. It stands out for its incredible speed and efficiency, capable of processing 100,000 time-series in seconds , thanks to its"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 528c8413-d41d-4fdb-97c0-8504a7cbfa91
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "splitters. It stands out for its incredible speed and efficiency, capable of processing 100,000 time-series in seconds , thanks to its smart use of Polars for parallel feature engineering. But functime is more than just speed; it's a powerful ally equipped with proven machine learning algorithms and supports exogenous features across all forecasters. It's also automated, managing lags and hyperparameter tuning with FLAML. With over 100 time-series feature extractors available in its custom ts namespace, functime offers an efficient, user-friendly platform that is transforming the way we approach time-series data."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 810358fa-3477-453a-a3e2-15f93a9a267e
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 a comprehensive library for time-series forecasting and feature extraction on large panel datasets. Offering unique preprocessing options and cross-validation splitters. It stands out for its incredible speed and efficiency, , thanks to its smart use of Polars for parallel feature engineering. But functime is more than just speed; it's a powerful ally equipped with proven machine learning algorithms and supports exogenous features across all forecasters. It's also automated, managing lags and hyperparameter tuning with FLAML. With over 100 time-series feature extractors available in its custom namespace, functime offers an efficient, user-friendly platform that is transforming the way we"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 30cc036a-a9b8-4724-9744-8c3962900d69
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "feature extractors available in its custom namespace, functime offers an efficient, user-friendly platform that is transforming the way we approach time-series data. LoRaX \u2014 an inference server that scales to 1000s of fine-tuned LLMs, by Predibase. LoRaX (\u201dLoRA eXchange\u201d) is designed to serve a multitude of fine-tuned models on a single GPU , significantly cutting down the costs associated with serving while maintaining impressive throughput and latency. Key features include Dynamic Adapter Loading, allowing just-in-time loading of any fine-tuned LoRA adapter without affecting concurrent requests; Heterogeneous Continuous Batching, which combines different adapter requests to maintain"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 467f0b86-2807-4741-a905-176f1ec95c6a
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "LoRA adapter without affecting concurrent requests; Heterogeneous Continuous Batching, which combines different adapter requests to maintain stable latency and throughput; and Adapter Exchange Scheduling, which efficiently manages adapter prefetching and offloading between GPU and CPU. It supports a range of Large Language Models like Llama and Mistral as base models, and is compatible with LoRA adapters trained using PEFT and Ludwig libraries."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: b87ffcc9-2f49-4b17-afc2-81e84d25860c
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 an inference server that scales to 1000s of fine-tuned LLMs, by Predibase. LoRaX (\u201dLoRA eXchange\u201d) is designed to serve a , significantly cutting down the costs associated with serving while maintaining impressive throughput and latency. Key features include Dynamic Adapter Loading, allowing just-in-time loading of any fine-tuned LoRA adapter without affecting concurrent requests; Heterogeneous Continuous Batching, which combines different adapter requests to maintain stable latency and throughput; and Adapter Exchange Scheduling, which efficiently manages adapter prefetching and offloading between GPU and CPU. It supports a range of Large Language Models like Llama and Mistral as base"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 950b029f-f15b-45eb-ba0b-b365ac628aaa
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "manages adapter prefetching and offloading between GPU and CPU. It supports a range of Large Language Models like Llama and Mistral as base models, and is compatible with LoRA adapters trained using PEFT and Ludwig libraries. outlines \u2014 guided text generation with LLMs, especially focusing on constrained text generation. Allows for model guidance such as, multiple choice, type constraints and dynamic stopping by using a very efficient regex-guided generation, leading to faster inference when compared to other guidance approaches. Generate valid JSON 100% of the time! Check out their HN Launch Blog post for more insights."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: ed04d99a-1f1c-4d68-ba5e-a8d5a74e8aca
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 with LLMs, especially focusing on constrained text generation. Allows for model guidance such as, multiple choice, type constraints and dynamic stopping by using a very efficient regex-guided generation, leading to faster inference when compared to other guidance approaches. Generate valid JSON 100% of the time! Check out their HN Launch Blog post for more insights. Pearl \u2014 a gift from Meta's Applied Reinforcement Learning (RL) team. It's a production-ready AI Agent Library that prioritizes cumulative long-term rewards over immediate feedback . These agents can adapt to environments with limited observability, sparse feedback, and high stochasticity, making them a fitting choice for a"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 09d4b531-0850-4ef3-b4af-256831158bcc
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": ". These agents can adapt to environments with limited observability, sparse feedback, and high stochasticity, making them a fitting choice for a wide range of real-world problems. Pearl's modular implementation of state-of-the-art RL algorithms is a testament to its versatility and forward-thinking approach."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 695d3f47-dd4c-4e70-941b-4817ec22da5e
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 a gift from Meta's Applied Reinforcement Learning (RL) team. It's a AI Agent Library that . These agents can adapt to environments with limited observability, sparse feedback, and high stochasticity, making them a fitting choice for a wide range of real-world problems. Pearl's modular implementation of state-of-the-art RL algorithms is a testament to its versatility and forward-thinking approach. PEFT \u2014 brought to you by Hugging Face, PEFT or Parameter-Efficient Fine-Tuning methods, is your ticket to fine-tuning pre-trained language models for a wide array of downstream applications without having to change all the model's parameters, saving on computational resources while also retaining"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: a684a8cf-f72b-41d2-b5c8-7eb52e54dc10
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "a wide array of downstream applications without having to change all the model's parameters, saving on computational resources while also retaining the good properties of more generalistic models and improving performance on your specific task."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d81a927b-a309-4d97-bb2b-92dd50170b69
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 brought to you by Hugging Face, PEFT or Parameter-Efficient Fine-Tuning methods, is your ticket to for a wide array of downstream applications without having to change all the model's parameters, saving on computational resources while also retaining the good properties of more generalistic models and improving performance on your specific task. vLLM \u2014 A high-throughput and memory-efficient inference and serving engine for LLMs. It improves LLM serving performance by 3.5x over previous SOTA, without changing model architecture. Since output size isn\u2019t known beforehand LLMs typically reserve very large and wasteful long blocks of contiguous memory as large as the model output size. The"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 4fc4f7c5-dc2c-49d6-94db-f69323ab56d3
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "size isn\u2019t known beforehand LLMs typically reserve very large and wasteful long blocks of contiguous memory as large as the model output size. The secret (actually not secret, it\u2019s open sourced) sauce is an algorithm called PagedAttention where they physically split the key-value caches that LLMs use into fixed length, short and dynamically allocated blocks. The dynamic assignment prevents memory waste, allowing more prompts to fit into VRAM and hence enabling faster throughput. At the time of writing, vLLM has been adopted as the serving solution by many of the big players, becoming a de-facto standard serving solution."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 618e1c4c-8dcc-4c99-96ff-546357b5e550
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "The Hidden Gems: Unveiling the Long Tail of 2023 Python Libraries\n\nWhile our top picks have certainly captured the limelight, there's a whole constellation of Python libraries that didn't make it to the main stage but are equally deserving of your attention. With more than 120 innovative Python libraries analyzed, each with its own unique capabilities and cutting-edge features, if we were to do justice to each one of them, we'd be here until the next Winter Olympics!\n\nTo help you navigate this treasure trove, we've neatly categorized these libraries and distilled their key features into succinct, one-liner summaries.\n\nCausal inference"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: f37f7ee0-e472-425b-b00d-cc2a3d902b09
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Causal inference\n\nCausalTune \u2014 a library for automated tuning and selection for causal estimators.\n\n\u2014 a library for automated tuning and selection for causal estimators. CausalPy \u2014 A Python package for causal inference in quasi-experimental settings.\n\n\u2014 A Python package for causal inference in quasi-experimental settings. PyWhy-LLM \u2014 experimental library integrating LLM capabilities to support causal analyses.\n\nChatblade \u2014 ChatGPT on the command line, providing utility methods to extract JSON or Markdown from ChatGPT responses.\n\n\u2014 ChatGPT on the command line, providing utility methods to extract JSON or Markdown from ChatGPT responses. Elia \u2014 A terminal ChatGPT client built with Textual."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 995834f0-bc02-41c7-b5c5-dfd0c81a0302
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 A terminal ChatGPT client built with Textual. Gorilla CLI \u2014 powers your command-line interactions with a user-centric tool. Simply state your objective, and Gorilla CLI will generate potential commands for execution.\n\n\u2014 powers your command-line interactions with a user-centric tool. Simply state your objective, and Gorilla CLI will generate potential commands for execution. LLM \u2014 A CLI utility and Python library for interacting with Large Language Models, both via remote APIs and models that can be installed and run on your own machine. By the author of Datasette.\n\nChainlit \u2014 \u201cthe Streamlit for ChatGPT\u201d, create ChatGPT-like UIs on top of any Python code in minutes!"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: cefbdd8f-e812-4b73-ad19-e131a750f87f
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Chainlit \u2014 \u201cthe Streamlit for ChatGPT\u201d, create ChatGPT-like UIs on top of any Python code in minutes!\n\n\u2014 \u201cthe Streamlit for ChatGPT\u201d, create ChatGPT-like UIs on top of any Python code in minutes! pydistcheck \u2014 Linter that finds portability issues in Python package distributions (wheels and sdists).\n\n\u2014 Linter that finds portability issues in Python package distributions (wheels and sdists). pyxet \u2014 lightweight interface for the XetHub platform, a blob-store with a filesystem like interface and git capabilities.\n\nComputer vision\n\ndeepdoctection \u2014 orchestrates document extraction and document layout analysis tasks using deep learning models."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d253f408-1b63-4f00-9236-ad1629b591d1
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Computer vision\n\ndeepdoctection \u2014 orchestrates document extraction and document layout analysis tasks using deep learning models.\n\n\u2014 orchestrates document extraction and document layout analysis tasks using deep learning models. FaceFusion \u2014 Next generation face swapper and enhancer.\n\n\u2014 Next generation face swapper and enhancer. MetaSeg \u2014 packaged version of the Segment Anything Model (SAM).\n\n\u2014 packaged version of the Segment Anything Model (SAM). VTracer \u2014 open source software to convert raster images (like jpg & png) into vector graphics (svg)\n\nData and Features\n\nAdala \u2014 Adala - Autonomous DAta (Labeling) Agent framework."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: c7f17171-d916-4baf-81ad-e7e33e1ba549
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Data and Features\n\nAdala \u2014 Adala - Autonomous DAta (Labeling) Agent framework.\n\n\u2014 Adala - Autonomous DAta (Labeling) Agent framework. Autolabel \u2014 Label, clean and enrich text datasets with LLMs.\n\n\u2014 Label, clean and enrich text datasets with LLMs. balance \u2014 simple workflow and methods for dealing with biased data samples when looking to infer from them to some target population of interest. See launch blog post. By META."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 10c676f6-57af-4128-a7c8-7861fe204a59
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 simple workflow and methods for dealing with biased data samples when looking to infer from them to some target population of interest. See launch blog post. By META. Bytewax \u2014 Python framework that simplifies event and stream processing. Because Bytewax couples the stream and event processing capabilities of Flink, Spark, and Kafka Streams with the friendly and familiar interface of Python, you can re-use the Python libraries you already know and love."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 5c6d9093-f9c7-4368-afcd-4468b6ea8f93
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 Python framework that simplifies event and stream processing. Because Bytewax couples the stream and event processing capabilities of Flink, Spark, and Kafka Streams with the friendly and familiar interface of Python, you can re-use the Python libraries you already know and love. Featureform \u2014 feature store. Turn your existing data infrastructure into a feature store.\n\n\u2014 feature store. Turn your existing data infrastructure into a feature store. Galactic cleaning and curation tools for massive unstructured text datasets. Ben (48/100) on X"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 8cd89b56-d936-4156-92db-139e446b60b0
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "cleaning and curation tools for massive unstructured text datasets. Ben (48/100) on X Great Expectations \u2014 helps data teams build a shared understanding of their data through quality testing, documentation, and profiling.\n\nData Visualization\n\nPyGWalker \u2014 turn your pandas DataFrame into a Tableau-style User Interface for visual analysis.\n\n\u2014 turn your pandas DataFrame into a Tableau-style User Interface for visual analysis. Vizro \u2014 a toolkit for creating modular data visualization applications. By McKinsey.\n\nEmbeddings and Vector DBs\n\nEpsilla \u2014 a high performance Vector Database Management System, focused on scalability, high performance, and cost-effectiveness of vector search."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 7adec506-2116-44bf-9489-c4f7bb2de582
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Epsilla \u2014 a high performance Vector Database Management System, focused on scalability, high performance, and cost-effectiveness of vector search.\n\n\u2014 a high performance Vector Database Management System, focused on scalability, high performance, and cost-effectiveness of vector search. LanceDB \u2014 open-source database for vector-search built with persistent storage, which greatly simplifies retrieval, filtering and management of embeddings."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 248c3e87-b07e-43a8-ab82-6c759682aadb
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 open-source database for vector-search built with persistent storage, which greatly simplifies retrieval, filtering and management of embeddings. SeaGOAT \u2014 local search tool that leverages vector embeddings to enable to search your codebase semantically.\n\n\u2014 local search tool that leverages vector embeddings to enable to search your codebase semantically. Text Embeddings Inference \u2014 A blazing fast inference solution for text embeddings models.\n\nFederated learning\n\nFlower \u2014 A Friendly Federated Learning Framework."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 298fee57-b352-4cad-bda8-70191bff452c
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Federated learning\n\nFlower \u2014 A Friendly Federated Learning Framework.\n\n\u2014 A Friendly Federated Learning Framework. MetisFL \u2014 federated learning framework that allows developers to easily federate their machine learning workflows and train their models across distributed data silos without ever collecting the data in a centralized location.\n\nGenerative AI\n\nAudioCraft \u2014 library for audio processing and generation with deep learning. By Meta.\n\n\u2014 library for audio processing and generation with deep learning. By Meta. Image Eval \u2014 A toolkit for evaluating your favorite image generation models. LinkedIn Launch Post."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 3be965c5-fda6-40cf-a1c0-5ddbc0d8cb88
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 A toolkit for evaluating your favorite image generation models. LinkedIn Launch Post. imaginAIry \u2014 Pythonic generation of stable diffusion images.\n\n\u2014 Pythonic generation of stable diffusion images. Modular Diffusion \u2014 Python library for designing and training your own Diffusion Models with PyTorch.\n\n\u2014 Python library for designing and training your own Diffusion Models with PyTorch. SapientML \u2014 Generative AutoML for Tabular Data.\n\nLLM Accuracy Enhancements\n\nAutoChain \u2014 AutoChain: Build lightweight, extensible, and testable LLM Agents\n\n\u2014 AutoChain: Build lightweight, extensible, and testable LLM Agents Auto-GPT \u2014 An experimental open-source attempt to make GPT-4 fully autonomous."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d86be4a1-af02-425a-87a0-4b2609523f9a
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 AutoChain: Build lightweight, extensible, and testable LLM Agents Auto-GPT \u2014 An experimental open-source attempt to make GPT-4 fully autonomous.\n\n\u2014 An experimental open-source attempt to make GPT-4 fully autonomous. Autotrain-Advanced \u2014 faster and easier training and deployments of state-of-the-art machine learning models.\n\n\u2014 faster and easier training and deployments of state-of-the-art machine learning models. DSPy \u2014 framework for solving advanced tasks with language models (LMs) and retrieval models (RMs). DSPy unifies techniques for prompting and fine-tuning LMs \u2014 and approaches for reasoning and tool/retrieval augmentation. By Stanford NLP."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 77df426d-aff8-48a4-95c3-370f1b137f3e
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 framework for solving advanced tasks with language models (LMs) and retrieval models (RMs). DSPy unifies techniques for prompting and fine-tuning LMs \u2014 and approaches for reasoning and tool/retrieval augmentation. By Stanford NLP. GPTCache \u2014 GPTCache is a library for creating semantic cache to store responses from LLM queries.\n\n\u2014 GPTCache is a library for creating semantic cache to store responses from LLM queries. Neural-Cherche \u2014 fine-tune neural search models such as Splade, ColBERT, and SparseEmbed on a specific dataset, and run efficient inference on a fine-tuned retriever or ranker."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 4b62d1fc-356d-4c26-8912-cbfe62ae2575
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 fine-tune neural search models such as Splade, ColBERT, and SparseEmbed on a specific dataset, and run efficient inference on a fine-tuned retriever or ranker. MemGPT \u2014 Teaching LLMs memory management for unbounded context \ud83d\udcda\ud83e\udd99.\n\n\u2014 Teaching LLMs memory management for unbounded context \ud83d\udcda\ud83e\udd99. nanoGPT \u2014 The simplest, fastest repository for training/finetuning medium-sized GPTs.\n\n\u2014 The simplest, fastest repository for training/finetuning medium-sized GPTs. Promptify \u2014 common prompts that work well to leverage LLMs for a variety of scenarios.\n\n\u2014 common prompts that work well to leverage LLMs for a variety of scenarios. SymbolicAI \u2014 Compositional Differentiable Programming Library."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 1423447c-7e19-437b-a436-4e76d6da85df
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 common prompts that work well to leverage LLMs for a variety of scenarios. SymbolicAI \u2014 Compositional Differentiable Programming Library.\n\n\u2014 Compositional Differentiable Programming Library. zep \u2014 a long-term memory store for LLM / Chatbot applications. Easily add relevant documents, chat history memory & rich user data to your LLM app's prompts.\n\nLLM App Building\n\nautollm \u2014 Ship RAG based LLM web apps in seconds.\n\n\u2014 Ship RAG based LLM web apps in seconds. Chidoriv \u2014 reactive runtime for building AI agents. It provides a framework for building AI agents that are reactive, observable, and robust. It supports building agents with Node.js, Python, and Rust."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 4683279d-ef92-4f1f-be17-4e54bb6a6157
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 reactive runtime for building AI agents. It provides a framework for building AI agents that are reactive, observable, and robust. It supports building agents with Node.js, Python, and Rust. FastChat \u2014 open platform for training, serving, and evaluating large language model based chatbots.\n\n\u2014 open platform for training, serving, and evaluating large language model based chatbots. GPTRouter \u2014 smoothly manage multiple LLMs and image models, speed up responses, and ensure non-stop reliability. Similar to LiteLLM, our top pick!"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 1fb68771-d6a0-40c3-b650-905e7f10b9a0
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 smoothly manage multiple LLMs and image models, speed up responses, and ensure non-stop reliability. Similar to LiteLLM, our top pick! guidance \u2014 a guidance language for controlling large language models.\n\n\u2014 a guidance language for controlling large language models. haystack \u2014 end-to-end NLP framework that enables you to build NLP applications powered by LLMs, Transformer models, vector search and more.\n\n\u2014 end-to-end NLP framework that enables you to build NLP applications powered by LLMs, Transformer models, vector search and more. Instructor \u2014 interact with OpenAI\u2019s function call API from Python code, with Python structs / objects."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 27e0894b-dc50-4adf-a335-eb01b0f49563
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 interact with OpenAI\u2019s function call API from Python code, with Python structs / objects. Jsonformer \u2014 A Bulletproof Way to Generate Structured JSON from Language Models\n\n\u2014 A Bulletproof Way to Generate Structured JSON from Language Models Langroid \u2014 easily build LLM-powered applications. Set up Agents, equip them with optional components (LLM, vector-store and methods), assign them tasks, and have them collaboratively solve a problem by exchanging messages."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: a850da41-9c6f-457e-ba06-bf17362a5537
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 easily build LLM-powered applications. Set up Agents, equip them with optional components (LLM, vector-store and methods), assign them tasks, and have them collaboratively solve a problem by exchanging messages. LLM App \u2014 build innovative AI applications by providing real-time human-like responses to user queries based on the most up-to-date knowledge available in your data sources.\n\n\u2014 build innovative AI applications by providing real-time human-like responses to user queries based on the most up-to-date knowledge available in your data sources. maccarone \u2014 AI-managed code blocks in Python, lets you delegate sections of your Python program to AI ownership."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 524878cc-b08a-4d96-89ce-33b7f8b12d0c
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 AI-managed code blocks in Python, lets you delegate sections of your Python program to AI ownership. magentic \u2014 prompt LLMs as simple Python functions using decorators.\n\n\u2014 prompt LLMs as simple Python functions using decorators. Semantic Kernel \u2014 integrate cutting-edge LLM technology quickly and easily into your apps. Microsoft\u2019s \u201cversion\u201d of LangChain.\n\naider \u2014 command line tool that lets you pair program with GPT-3.5/GPT-4, to edit code stored in your local git repository.\n\n\u2014 command line tool that lets you pair program with GPT-3.5/GPT-4, to edit code stored in your local git repository. ChatGDB \u2014 Harness the power of ChatGPT inside the GDB debugger!"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: a618c56c-6753-4bde-a266-ee33fe490014
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 Harness the power of ChatGPT inside the GDB debugger! Dataherald \u2014 natural language-to-SQL engine built for enterprise-level question answering over structured data. HN launch post.\n\n\u2014 natural language-to-SQL engine built for enterprise-level question answering over structured data. HN launch post. FauxPilot \u2014 open-source GitHub Copilot server.\n\n\u2014 open-source GitHub Copilot server. GPT Engineer \u2014 Specify what you want it to build, the AI asks for clarification, and then builds it."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: f4c0af9d-4ce1-489d-a0be-2c00b0c13c61
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 open-source GitHub Copilot server. GPT Engineer \u2014 Specify what you want it to build, the AI asks for clarification, and then builds it.\n\n\u2014 Specify what you want it to build, the AI asks for clarification, and then builds it. gpt-repository-loader \u2014 command-line tool that converts the contents of a Git repository into a text format that can be interpreted by LLMs.\n\n\u2014 command-line tool that converts the contents of a Git repository into a text format that can be interpreted by LLMs. ipython-gpt \u2014 extension that allows you to use ChatGPT directly from your Jupyter Notebook or IPython Shell."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 2fd7a6bf-d9f8-455c-8f00-18f8feb6df3a
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 extension that allows you to use ChatGPT directly from your Jupyter Notebook or IPython Shell. Jupyter AI \u2014 generative AI extension for JupyterLab.\n\n\u2014 generative AI extension for JupyterLab. PlotAI \u2014 use ChatGPT to create plots in Python and Matplotlib directly in your Python script or notebook.\n\n\u2014 use ChatGPT to create plots in Python and Matplotlib directly in your Python script or notebook. sketch \u2014 AI code-writing assistant for pandas users that understands the context of your data, greatly improving the relevance of suggestions.\n\nLLM Development\n\ndistilabel \u2014 AI Feedback framework for scalable LLM alignment."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 6ca19552-5ed1-4866-a064-3258545e244e
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "LLM Development\n\ndistilabel \u2014 AI Feedback framework for scalable LLM alignment.\n\n\u2014 AI Feedback framework for scalable LLM alignment. language-model-arithmetic \u2014 controlled text generation via language model arithmetic.\n\n\u2014 controlled text generation via language model arithmetic. Lit-GPT \u2014 Hackable implementation of state-of-the-art open-source LLMs based on nanoGPT. Supports flash attention, 4-bit and 8-bit quantization, LoRA and LLaMA-Adapter fine-tuning, pre-training."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 0c4d7ed4-dd3b-44d0-90b3-321871b762fc
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 Hackable implementation of state-of-the-art open-source LLMs based on nanoGPT. Supports flash attention, 4-bit and 8-bit quantization, LoRA and LLaMA-Adapter fine-tuning, pre-training. Lit-LLaMA \u2014 Implementation of the LLaMA language model based on nanoGPT. Supports flash attention, Int8 and GPTQ 4bit quantization, LoRA and LLaMA-Adapter fine-tuning, pre-training.\n\n\u2014 Implementation of the LLaMA language model based on nanoGPT. Supports flash attention, Int8 and GPTQ 4bit quantization, LoRA and LLaMA-Adapter fine-tuning, pre-training. LMQL \u2014 a query language for programming (large) language models.\n\nLLM Experimentation"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 922aac29-c3fa-4f77-921a-53cd1ca23ce5
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "LLM Experimentation\n\nChainForge \u2014 open-source visual programming environment for battle-testing prompts to LLMs.\n\n\u2014 open-source visual programming environment for battle-testing prompts to LLMs. Langflow \u2014 UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows.\n\n\u2014 UI for LangChain, designed with react-flow to provide an effortless way to experiment and prototype flows. PromptTools \u2014 a set of open-source, self-hostable tools for experimenting with, testing, and evaluating LLMs, vector databases, and prompts. HN launch post.\n\nLLM Metrics\n\nDeepEval \u2014 a simple-to-use, open-source evaluation framework for LLM applications."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 4b65b427-805a-416f-b570-e13b6b02f43c
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "LLM Metrics\n\nDeepEval \u2014 a simple-to-use, open-source evaluation framework for LLM applications.\n\n\u2014 a simple-to-use, open-source evaluation framework for LLM applications. Fiddler Auditor \u2014 a tool to evaluate the robustness of language models.\n\n\u2014 a tool to evaluate the robustness of language models. ragas \u2014 Evaluation framework for your Retrieval Augmented Generation (RAG) pipelines.\n\n\u2014 Evaluation framework for your Retrieval Augmented Generation (RAG) pipelines. tvalmetrics \u2014 Metrics to evaluate the quality of responses of your Retrieval Augmented Generation (RAG) applications.\n\nLLM Serving"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 934ead08-4a66-4004-ada2-293a415a7fac
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "LLM Serving\n\nAviary \u2014 an LLM serving solution that makes it easy to deploy and manage a variety of open source LLMs. By the authors of Ray.\n\n\u2014 an LLM serving solution that makes it easy to deploy and manage a variety of open source LLMs. By the authors of Ray. GPT4All \u2014 an ecosystem to run powerful and customized large language models that work locally on consumer grade CPUs and any GPU (ex pygpt4all/pyllamacpp) with python bindings.\n\n\u2014 an ecosystem to run powerful and customized large language models that work locally on consumer grade CPUs and any GPU (ex pygpt4all/pyllamacpp) with python bindings. LLM Engine \u2014 engine for fine-tuning and serving large language models. By Scale AI."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 8f4fa4cd-99ff-44f5-8d03-4eac13f50208
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 engine for fine-tuning and serving large language models. By Scale AI. LLM Gateway \u2014 gateway for secure & reliable communications with OpenAI and other LLM providers.\n\n\u2014 gateway for secure & reliable communications with OpenAI and other LLM providers. punica \u2014 Serving multiple LoRA finetuned LLM as one.\n\n\u2014 Serving multiple LoRA finetuned LLM as one. Ollama \u2014 Get up and running with Llama 2 and other large language models locally.\n\n\u2014 Get up and running with Llama 2 and other large language models locally. OnPrem.LLM \u2014 tool for running on-premises large language models with non-public data."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 4b623cd9-9ca3-4320-a2bc-d0c252b2f070
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 tool for running on-premises large language models with non-public data. OpenLLM \u2014 An open platform for operating large language models (LLMs) in production. Fine-tune, serve, deploy, and monitor any LLMs with ease. By BentoML.\n\n\u2014 An open platform for operating large language models (LLMs) in production. Fine-tune, serve, deploy, and monitor any LLMs with ease. By BentoML. OpenLLMetry \u2014 Open-source observability for your LLM application, based on OpenTelemetry.\n\n\u2014 Open-source observability for your LLM application, based on OpenTelemetry. privateGPT \u2014 Interact privately with your documents using the power of GPT, 100% privately, no data leaks."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 4c7b583b-ef91-49ce-8e35-c67785e92945
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "IncarnaMind \u2014 Connect and chat with your multiple documents (pdf and txt) through GPT and Claude LLMs in a minute.\n\n\u2014 Connect and chat with your multiple documents (pdf and txt) through GPT and Claude LLMs in a minute. Puncia \u2014 leveraging AI and other tools, it will tell you everything about a web domain or subdomain, like finding hidden subdomains.\n\n\u2014 leveraging AI and other tools, it will tell you everything about a web domain or subdomain, like finding hidden subdomains. scrapeghost \u2014 experimental library for scraping websites using OpenAI's GPT API.\n\nMLOps, LLMOps, DevOps"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 9e3b91f3-465c-4b0e-babb-e2fecc4581db
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "MLOps, LLMOps, DevOps\n\nphoenix \u2014 ML Observability in a Notebook - Uncover Insights, Surface Problems, Monitor, and Fine Tune your Generative LLM, CV and Tabular Models.\n\nLLaVAv \u2014 Visual Instruction Tuning - Large Language-and-Vision Assistant built towards multimodal GPT-4 level capabilities.\n\n\u2014 Visual Instruction Tuning - Large Language-and-Vision Assistant built towards multimodal GPT-4 level capabilities. Multimodal-Maestro \u2014 effective prompting for Large Multimodal Models like GPT-4 Vision, LLaVA or CogVLM.\n\n\u2014 effective prompting for Large Multimodal Models like GPT-4 Vision, LLaVA or CogVLM. Nougat \u2014 the academic document PDF parser that understands LaTeX math and tables."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 0efc9b06-78af-441f-852a-e4ef6b5daf13
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 the academic document PDF parser that understands LaTeX math and tables. UForm \u2014 Pocket-Sized Multi-Modal AI For Semantic Search & Recommendation Systems.\n\nPython ML\n\ndifflogic \u2014 A Library for Differentiable Logic Gate Networks by Felix Petersen.\n\n\u2014 A Library for Differentiable Logic Gate Networks by Felix Petersen. TensorDict \u2014 a dictionary-like class that inherits properties from tensors, such as indexing, shape operations, casting to device etc. The main purpose of TensorDict is to make code-bases more readable and modular by abstracting away tailored operations.\n\nPerformance and scalability"}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 419dfd8d-ae97-4a40-90b3-37d3cae646ef
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "Performance and scalability\n\nAITemplate \u2014 Python framework which renders neural network into high performance CUDA/HIP C++ code. Specialized for FP16 TensorCore (NVIDIA GPU) and MatrixCore (AMD GPU) inference.\n\n\u2014 Python framework which renders neural network into high performance CUDA/HIP C++ code. Specialized for FP16 TensorCore (NVIDIA GPU) and MatrixCore (AMD GPU) inference. AutoGPTQ \u2014 easy-to-use LLMs quantization package with user-friendly APIs, based on GPTQ algorithm."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: 8c1cf1cc-73bf-41f9-b1cb-a6a3d2526fdd
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 easy-to-use LLMs quantization package with user-friendly APIs, based on GPTQ algorithm. composer \u2014 PyTorch library that enables you to train neural networks faster, at lower cost, and to higher accuracy. Implements more than two dozen speedup methods that can be applied to your training loop in just a few lines of code.\n\n\u2014 PyTorch library that enables you to train neural networks faster, at lower cost, and to higher accuracy. Implements more than two dozen speedup methods that can be applied to your training loop in just a few lines of code. fastLLaMa \u2014 Python wrapper to run Inference of LLaMA models using C++."}


created_at: 2024-05-13 19:07:57
topic: persistent://default/default/3c4faa6d-6b2b-430c-8727-e0594beeca70
id: d0789615-b06c-4d43-ace1-f91ed88b6b73
encoding: FLOAT32
metadata:
{"title": "Top Python libraries of 2023", "link": "https://tryolabs.com/blog/top-python-libraries-2023", "language": "en", "description": "Exploring the expansive universe of Python libraries in 2023, our curated selection goes beyond the usual suspects to uncover hidden gems and innovative tools. Dive in!", "chroma:document": "\u2014 Python wrapper to run Inference of LLaMA models using C++. hidet \u2014 open-source deep learning compiler, written in Python. It supports end-to-end compilation of DNN models from PyTorch and ONNX to efficient cuda kernels.\n\n\u2014 open-source deep learning compiler, written in Python. It supports end-to-end compilation of DNN models from PyTorch and ONNX to efficient cuda kernels. LPython \u2014 compiler that aggressively optimizes type-annotated Python code. It has several backends, including LLVM, C, C++, and WASM. LPython\u2019s primary tenet is speed. Launch blog post."}


