Added AI skills
This commit is contained in:
153
.agents/skills/zod/references/perf-arrays.md
Normal file
153
.agents/skills/zod/references/perf-arrays.md
Normal file
@@ -0,0 +1,153 @@
|
||||
---
|
||||
title: Optimize Large Array Validation
|
||||
impact: LOW-MEDIUM
|
||||
impactDescription: Validating 10,000 items takes ~100ms; early exits, sampling, or batching reduce time for large datasets
|
||||
tags: perf, arrays, batch, large-data
|
||||
---
|
||||
|
||||
## Optimize Large Array Validation
|
||||
|
||||
Validating large arrays (thousands of items) can become a performance bottleneck. For batch imports, streaming data, or large datasets, consider strategies like early exit, sampling, or batched validation.
|
||||
|
||||
**Baseline performance:**
|
||||
|
||||
```typescript
|
||||
import { z } from 'zod'
|
||||
|
||||
const itemSchema = z.object({
|
||||
id: z.string(),
|
||||
value: z.number(),
|
||||
})
|
||||
|
||||
const arraySchema = z.array(itemSchema)
|
||||
|
||||
// 10,000 items: ~100ms
|
||||
// 100,000 items: ~1000ms
|
||||
arraySchema.parse(largeArray)
|
||||
```
|
||||
|
||||
**Early exit on first error:**
|
||||
|
||||
```typescript
|
||||
import { z } from 'zod'
|
||||
|
||||
function validateArrayFastFail<T>(
|
||||
schema: z.ZodType<T>,
|
||||
items: unknown[]
|
||||
): { success: true; data: T[] } | { success: false; error: z.ZodError; index: number } {
|
||||
const validated: T[] = []
|
||||
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
const result = schema.safeParse(items[i])
|
||||
if (!result.success) {
|
||||
return { success: false, error: result.error, index: i }
|
||||
}
|
||||
validated.push(result.data)
|
||||
}
|
||||
|
||||
return { success: true, data: validated }
|
||||
}
|
||||
|
||||
// Stops at first invalid item instead of validating all
|
||||
```
|
||||
|
||||
**Sample validation for large datasets:**
|
||||
|
||||
```typescript
|
||||
function validateSample<T>(
|
||||
schema: z.ZodType<T>,
|
||||
items: unknown[],
|
||||
sampleSize: number = 100
|
||||
): { valid: boolean; sampleErrors?: z.ZodIssue[] } {
|
||||
// Validate random sample
|
||||
const indices = new Set<number>()
|
||||
while (indices.size < Math.min(sampleSize, items.length)) {
|
||||
indices.add(Math.floor(Math.random() * items.length))
|
||||
}
|
||||
|
||||
const errors: z.ZodIssue[] = []
|
||||
|
||||
for (const i of indices) {
|
||||
const result = schema.safeParse(items[i])
|
||||
if (!result.success) {
|
||||
errors.push(...result.error.issues)
|
||||
}
|
||||
}
|
||||
|
||||
return errors.length > 0
|
||||
? { valid: false, sampleErrors: errors }
|
||||
: { valid: true }
|
||||
}
|
||||
|
||||
// Check 100 random items from 100,000 - very fast
|
||||
const check = validateSample(itemSchema, hugeArray)
|
||||
```
|
||||
|
||||
**Batched validation with progress:**
|
||||
|
||||
```typescript
|
||||
async function validateInBatches<T>(
|
||||
schema: z.ZodType<T>,
|
||||
items: unknown[],
|
||||
batchSize: number = 1000,
|
||||
onProgress?: (percent: number) => void
|
||||
): Promise<z.SafeParseReturnType<unknown, T[]>> {
|
||||
const validated: T[] = []
|
||||
const errors: z.ZodIssue[] = []
|
||||
|
||||
for (let i = 0; i < items.length; i += batchSize) {
|
||||
const batch = items.slice(i, i + batchSize)
|
||||
|
||||
// Validate batch
|
||||
for (let j = 0; j < batch.length; j++) {
|
||||
const result = schema.safeParse(batch[j])
|
||||
if (result.success) {
|
||||
validated.push(result.data)
|
||||
} else {
|
||||
errors.push(...result.error.issues.map(issue => ({
|
||||
...issue,
|
||||
path: [i + j, ...issue.path],
|
||||
})))
|
||||
}
|
||||
}
|
||||
|
||||
// Report progress and yield to event loop
|
||||
onProgress?.(Math.min(100, ((i + batchSize) / items.length) * 100))
|
||||
await new Promise(resolve => setTimeout(resolve, 0))
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
return { success: false, error: new z.ZodError(errors) }
|
||||
}
|
||||
return { success: true, data: validated }
|
||||
}
|
||||
|
||||
// Use with progress reporting
|
||||
await validateInBatches(itemSchema, largeArray, 1000, (percent) => {
|
||||
console.log(`Validating: ${percent.toFixed(1)}%`)
|
||||
})
|
||||
```
|
||||
|
||||
**Streaming validation:**
|
||||
|
||||
```typescript
|
||||
async function* validateStream<T>(
|
||||
schema: z.ZodType<T>,
|
||||
items: AsyncIterable<unknown>
|
||||
): AsyncGenerator<T, void, unknown> {
|
||||
for await (const item of items) {
|
||||
yield schema.parse(item) // Throws on invalid
|
||||
}
|
||||
}
|
||||
|
||||
// Process items as they arrive
|
||||
for await (const validItem of validateStream(itemSchema, dataStream)) {
|
||||
await processItem(validItem)
|
||||
}
|
||||
```
|
||||
|
||||
**When NOT to use this pattern:**
|
||||
- Small arrays (< 1000 items) - standard validation is fine
|
||||
- When all items must be validated for correctness guarantees
|
||||
|
||||
Reference: [Zod Performance](https://zod.dev/v4#performance)
|
||||
Reference in New Issue
Block a user