Merge pull request #5417 from ywk253100/180716_redis_cross_slot

Fix cross slot issue of redis
This commit is contained in:
Steven Zou 2018-07-30 13:50:41 +08:00 committed by GitHub
commit 9f37a66f21
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
109 changed files with 13322 additions and 477 deletions

15
src/Gopkg.lock generated
View File

@ -165,8 +165,8 @@
[[projects]]
name = "github.com/gocraft/work"
packages = ["."]
revision = "67c8dc210ce647e8d8a4206c7f5856ea2390f290"
version = "v0.5.0"
revision = "1d4117a214abff263b472043871c8666aedb716b"
version = "v0.5.1"
[[projects]]
name = "github.com/golang-migrate/migrate"
@ -189,6 +189,15 @@
]
revision = "130e6b02ab059e7b717a096f397c5b60111cae74"
[[projects]]
name = "github.com/gomodule/redigo"
packages = [
"internal",
"redis"
]
revision = "9c11da706d9b7902c6da69c592f75637793fe121"
version = "v2.0.0"
[[projects]]
branch = "master"
name = "github.com/google/go-querystring"
@ -372,6 +381,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "6849737bf604451aa7abcb3a56fca93cdf7bc9cb7da41e7d5aa796701cea559a"
inputs-digest = "f2bc787303dc9f72125800ed44f5fefed9e832183419e563183e87be26534a1c"
solver-name = "gps-cdcl"
solver-version = 1

View File

@ -99,3 +99,7 @@ ignored = ["github.com/vmware/harbor/tests*"]
[[constraint]]
name = "github.com/Masterminds/semver"
version = "=1.4.2"
[[constraint]]
name = "github.com/gocraft/work"
version = "=0.5.1"

View File

@ -10,7 +10,7 @@ import (
"sync"
"time"
"github.com/garyburd/redigo/redis"
"github.com/gomodule/redigo/redis"
"github.com/vmware/harbor/src/jobservice/logger"
"github.com/vmware/harbor/src/jobservice/models"
"github.com/vmware/harbor/src/jobservice/utils"

View File

@ -16,7 +16,7 @@ import (
"github.com/vmware/harbor/src/jobservice/errs"
"github.com/vmware/harbor/src/jobservice/logger"
"github.com/garyburd/redigo/redis"
"github.com/gomodule/redigo/redis"
"github.com/vmware/harbor/src/jobservice/job"
"github.com/vmware/harbor/src/jobservice/models"
"github.com/vmware/harbor/src/jobservice/utils"

View File

@ -12,7 +12,7 @@ import (
"testing"
"time"
"github.com/garyburd/redigo/redis"
"github.com/gomodule/redigo/redis"
"github.com/vmware/harbor/src/jobservice/job"
"github.com/vmware/harbor/src/jobservice/models"
"github.com/vmware/harbor/src/jobservice/utils"

View File

@ -6,8 +6,8 @@ import (
"math/rand"
"time"
"github.com/garyburd/redigo/redis"
"github.com/gocraft/work"
"github.com/gomodule/redigo/redis"
"github.com/robfig/cron"
"github.com/vmware/harbor/src/jobservice/job"
"github.com/vmware/harbor/src/jobservice/logger"

View File

@ -13,7 +13,7 @@ import (
"github.com/robfig/cron"
"github.com/garyburd/redigo/redis"
"github.com/gomodule/redigo/redis"
"github.com/vmware/harbor/src/jobservice/env"
"github.com/vmware/harbor/src/jobservice/logger"
"github.com/vmware/harbor/src/jobservice/models"

View File

@ -8,7 +8,7 @@ import (
"github.com/gocraft/work"
"github.com/garyburd/redigo/redis"
"github.com/gomodule/redigo/redis"
"github.com/vmware/harbor/src/jobservice/logger"
"github.com/vmware/harbor/src/jobservice/utils"
)

View File

@ -14,7 +14,7 @@ import (
"github.com/vmware/harbor/src/jobservice/opm"
"github.com/vmware/harbor/src/jobservice/period"
"github.com/garyburd/redigo/redis"
"github.com/gomodule/redigo/redis"
"github.com/vmware/harbor/src/jobservice/models"
"github.com/vmware/harbor/src/jobservice/utils"
)

View File

@ -8,8 +8,8 @@ import (
"math"
"time"
"github.com/garyburd/redigo/redis"
"github.com/gocraft/work"
"github.com/gomodule/redigo/redis"
"github.com/robfig/cron"
"github.com/vmware/harbor/src/jobservice/env"
"github.com/vmware/harbor/src/jobservice/job"

View File

@ -4,13 +4,14 @@ package runtime
import (
"context"
"fmt"
"os"
"os/signal"
"sync"
"syscall"
"time"
"github.com/garyburd/redigo/redis"
"github.com/gomodule/redigo/redis"
"github.com/vmware/harbor/src/common/job"
"github.com/vmware/harbor/src/jobservice/api"
"github.com/vmware/harbor/src/jobservice/config"
@ -173,7 +174,7 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool(ctx *env.Context, cfg *config.Con
}
redisWorkerPool := pool.NewGoCraftWorkPool(ctx,
cfg.PoolConfig.RedisPoolCfg.Namespace,
fmt.Sprintf("{%s}", cfg.PoolConfig.RedisPoolCfg.Namespace),
cfg.PoolConfig.WorkerCount,
redisPool)
//Register jobs here

View File

@ -9,7 +9,7 @@ import (
"os"
"time"
"github.com/garyburd/redigo/redis"
"github.com/gomodule/redigo/redis"
)
const (

View File

@ -11,7 +11,7 @@ import (
"strconv"
"strings"
"github.com/garyburd/redigo/redis"
"github.com/gomodule/redigo/redis"
)
//IsEmptyStr check if the specified str is empty (len ==0) after triming prefix and suffix spaces.

View File

@ -10,6 +10,7 @@ gocraft/work lets you enqueue and processes background jobs in Go. Jobs are dura
* Enqueue unique jobs so that only one job with a given name/arguments exists in the queue at once.
* Web UI to manage failed jobs and observe the system.
* Periodically enqueue jobs on a cron-like schedule.
* Pause / unpause jobs and control concurrency within and across processes
## Enqueue new jobs
@ -19,7 +20,7 @@ To enqueue jobs, you need to make an Enqueuer with a redis namespace and a redig
package main
import (
"github.com/garyburd/redigo/redis"
"github.com/gomodule/redigo/redis"
"github.com/gocraft/work"
)
@ -55,7 +56,7 @@ In order to process jobs, you'll need to make a WorkerPool. Add middleware and j
package main
import (
"github.com/garyburd/redigo/redis"
"github.com/gomodule/redigo/redis"
"github.com/gocraft/work"
"os"
"os/signal"
@ -91,7 +92,7 @@ func main() {
pool.Job("send_email", (*Context).SendEmail)
// Customize options:
pool.JobWithOptions("export", JobOptions{Priority: 10, MaxFails: 1}, (*Context).Export)
pool.JobWithOptions("export", work.JobOptions{Priority: 10, MaxFails: 1}, (*Context).Export)
// Start processing jobs
pool.Start()
@ -217,7 +218,7 @@ go install github.com/gocraft/work/cmd/workwebui
Then, you can run it:
```bash
workwebui -redis=":6379" -ns="work" -listen=":5040"
workwebui -redis="redis:6379" -ns="work" -listen=":5040"
```
Navigate to ```http://localhost:5040/```.
@ -245,10 +246,11 @@ You'll see a view that looks like this:
### Processing a job
* To process a job, a worker will execute a Lua script to atomically move a job its queue to an in-progress queue.
* The worker will then run the job. The job will either finish successfully or result in an error or panic.
* A job is dequeued and moved to in-progress if the job queue is not paused and the number of active jobs does not exceed concurrency limit for the job type
* The worker will then run the job and increment the job lock. The job will either finish successfully or result in an error or panic.
* If the process completely crashes, the reaper will eventually find it in its in-progress queue and requeue it.
* If the job is successful, we'll simply remove the job from the in-progress queue.
* If the job returns an error or panic, we'll see how many retries a job has left. If it doesn't have any, we'll move it to the dead queue. If it has retries left, we'll consume a retry and add the job to the retry queue.
* If the job returns an error or panic, we'll see how many retries a job has left. If it doesn't have any, we'll move it to the dead queue. If it has retries left, we'll consume a retry and add the job to the retry queue.
### Workers and WorkerPools
@ -283,12 +285,29 @@ You'll see a view that looks like this:
* When a unique job is enqueued, we'll atomically set a redis key that includes the job name and arguments and enqueue the job.
* When the job is processed, we'll delete that key to permit another job to be enqueued.
### Periodic Jobs
### Periodic jobs
* You can tell a worker pool to enqueue jobs periodically using a cron schedule.
* Each worker pool will wake up every 2 minutes, and if jobs haven't been scheduled yet, it will schedule all the jobs that would be executed in the next five minutes.
* Each periodic job that runs at a given time has a predictable byte pattern. Since jobs are scheduled on the scheduled job queue (a Redis z-set), if the same job is scheduled twice for a given time, it can only exist in the z-set once.
## Paused jobs
* You can pause jobs from being processed from a specific queue by setting a "paused" redis key (see `redisKeyJobsPaused`)
* Conversely, jobs in the queue will resume being processed once the paused redis key is removed
## Job concurrency
* You can control job concurrency using `JobOptions{MaxConcurrency: <num>}`.
* Unlike the WorkerPool concurrency, this controls the limit on the number jobs of that type that can be active at one time by within a single redis instance
* This works by putting a precondition on enqueuing function, meaning a new job will not be scheduled if we are at or over a job's `MaxConcurrency` limit
* A redis key (see `redisKeyJobsLock`) is used as a counting semaphore in order to track job concurrency per job type
* The default value is `0`, which means "no limit on job concurrency"
* **Note:** if you want to run jobs "single threaded" then you can set the `MaxConcurrency` accordingly:
```go
worker_pool.JobWithOptions(jobName, JobOptions{MaxConcurrency: 1}, (*Context).WorkFxn)
```
### Terminology reference
* "worker pool" - a pool of workers
* "worker" - an individual worker in a single goroutine. Gets a job from redis, does job, gets next job...
@ -301,10 +320,11 @@ You'll see a view that looks like this:
* "job name" - each job has a name, like "create_watch"
* "job type" - backend/private nomenclature for the handler+options for processing a job
* "queue" - each job creates a queue with the same name as the job. only jobs named X go into the X queue.
* "retry jobs" - If a job fails and needs to be retried, it will be put on this queue.
* "scheduled jobs" - Jobs enqueued to be run in th future will be put on a scheduled job queue.
* "dead jobs" - If a job exceeds its MaxFails count, it will be put on the dead job queue.
* "retry jobs" - if a job fails and needs to be retried, it will be put on this queue.
* "scheduled jobs" - jobs enqueued to be run in th future will be put on a scheduled job queue.
* "dead jobs" - if a job exceeds its MaxFails count, it will be put on the dead job queue.
* "paused jobs" - if paused key is present for a queue, then no jobs from that queue will be processed by any workers until that queue's paused key is removed
* "job concurrency" - the number of jobs being actively processed of a particular type across worker pool processes but within a single redis instance
## Benchmarks
@ -333,5 +353,4 @@ These packages were developed by the [engineering team](https://eng.uservoice.co
* Jonathan Novak -- [https://github.com/cypriss](https://github.com/cypriss)
* Tai-Lin Chu -- [https://github.com/taylorchu](https://github.com/taylorchu)
* Tyler Smith -- [https://github.com/tyler-smith](https://github.com/tyler-smith)
* Sponsored by [UserVoice](https://eng.uservoice.com)

View File

@ -2,12 +2,13 @@ package main
import (
"fmt"
"github.com/benmanns/goworker"
"github.com/garyburd/redigo/redis"
"github.com/gocraft/health"
"os"
"sync/atomic"
"time"
"github.com/benmanns/goworker"
"github.com/gocraft/health"
"github.com/gomodule/redigo/redis"
)
func myJob(queue string, args ...interface{}) error {

View File

@ -2,12 +2,13 @@ package main
import (
"fmt"
"github.com/garyburd/redigo/redis"
"github.com/gocraft/health"
"github.com/jrallison/go-workers"
"os"
"sync/atomic"
"time"
"github.com/gocraft/health"
"github.com/gomodule/redigo/redis"
"github.com/jrallison/go-workers"
)
func myJob(m *workers.Msg) {

View File

@ -2,12 +2,13 @@ package main
import (
"fmt"
"github.com/albrow/jobs"
"github.com/garyburd/redigo/redis"
"github.com/gocraft/health"
"os"
"sync/atomic"
"time"
"github.com/albrow/jobs"
"github.com/gocraft/health"
"github.com/gomodule/redigo/redis"
)
var namespace = "jobs"
@ -51,8 +52,8 @@ func main() {
job = stream.NewJob("run_all")
pool, err := jobs.NewPool(&jobs.PoolConfig{
// NumWorkers: 1000,
// BatchSize: 3000,
// NumWorkers: 1000,
// BatchSize: 3000,
})
if err != nil {
panic(err)

View File

@ -2,12 +2,13 @@ package main
import (
"fmt"
"github.com/garyburd/redigo/redis"
"github.com/gocraft/health"
"github.com/gocraft/work"
"os"
"sync/atomic"
"time"
"github.com/gocraft/health"
"github.com/gocraft/work"
"github.com/gomodule/redigo/redis"
)
var namespace = "bench_test"

View File

@ -2,10 +2,11 @@ package work
import (
"fmt"
"github.com/garyburd/redigo/redis"
"sort"
"strconv"
"strings"
"github.com/gomodule/redigo/redis"
)
// ErrNotDeleted is returned by functions that delete jobs to indicate that although the redis commands were successful,

View File

@ -2,10 +2,11 @@ package work
import (
"fmt"
"github.com/garyburd/redigo/redis"
"github.com/stretchr/testify/assert"
"testing"
"time"
"github.com/gomodule/redigo/redis"
"github.com/stretchr/testify/assert"
)
type TestContext struct{}

View File

@ -4,10 +4,11 @@ import (
"encoding/json"
"flag"
"fmt"
"github.com/garyburd/redigo/redis"
"github.com/gocraft/work"
"os"
"time"
"github.com/gocraft/work"
"github.com/gomodule/redigo/redis"
)
var redisHostPort = flag.String("redis", ":6379", "redis hostport")

View File

@ -3,10 +3,11 @@ package main
import (
"flag"
"fmt"
"github.com/garyburd/redigo/redis"
"github.com/gocraft/work"
"math/rand"
"time"
"github.com/gocraft/work"
"github.com/gomodule/redigo/redis"
)
var redisHostPort = flag.String("redis", ":6379", "redis hostport")

View File

@ -8,8 +8,8 @@ import (
"strconv"
"time"
"github.com/garyburd/redigo/redis"
"github.com/gocraft/work/webui"
"github.com/gomodule/redigo/redis"
)
var (
@ -55,7 +55,7 @@ func newPool(addr string, database int) *redis.Pool {
MaxIdle: 3,
IdleTimeout: 240 * time.Second,
Dial: func() (redis.Conn, error) {
return redis.Dial("tcp", addr, redis.DialDatabase(database))
return redis.DialURL(addr, redis.DialDatabase(database))
},
Wait: true,
}

View File

@ -6,25 +6,34 @@ import (
"strings"
"time"
"github.com/garyburd/redigo/redis"
"github.com/gomodule/redigo/redis"
)
const (
deadTime = 5 * time.Minute
reapPeriod = 10 * time.Minute
deadTime = 10 * time.Second // 2 x heartbeat
reapPeriod = 10 * time.Minute
reapJitterSecs = 30
requeueKeysPerJob = 4
)
type deadPoolReaper struct {
namespace string
pool *redis.Pool
namespace string
pool *redis.Pool
deadTime time.Duration
reapPeriod time.Duration
curJobTypes []string
stopChan chan struct{}
doneStoppingChan chan struct{}
}
func newDeadPoolReaper(namespace string, pool *redis.Pool) *deadPoolReaper {
func newDeadPoolReaper(namespace string, pool *redis.Pool, curJobTypes []string) *deadPoolReaper {
return &deadPoolReaper{
namespace: namespace,
pool: pool,
deadTime: deadTime,
reapPeriod: reapPeriod,
curJobTypes: curJobTypes,
stopChan: make(chan struct{}),
doneStoppingChan: make(chan struct{}),
}
@ -40,13 +49,8 @@ func (r *deadPoolReaper) stop() {
}
func (r *deadPoolReaper) loop() {
// Reap
if err := r.reap(); err != nil {
logError("dead_pool_reaper.reap", err)
}
// Begin reaping periodically
timer := time.NewTimer(reapPeriod)
// Reap immediately after we provide some time for initialization
timer := time.NewTimer(r.deadTime)
defer timer.Stop()
for {
@ -55,8 +59,8 @@ func (r *deadPoolReaper) loop() {
r.doneStoppingChan <- struct{}{}
return
case <-timer.C:
// Schedule next occurrence with jitter
timer.Reset(reapPeriod + time.Duration(rand.Intn(30))*time.Second)
// Schedule next occurrence periodically with jitter
timer.Reset(r.reapPeriod + time.Duration(rand.Intn(reapJitterSecs))*time.Second)
// Reap
if err := r.reap(); err != nil {
@ -80,18 +84,23 @@ func (r *deadPoolReaper) reap() error {
// Cleanup all dead pools
for deadPoolID, jobTypes := range deadPoolIDs {
// Requeue all dangling jobs
r.requeueInProgressJobs(deadPoolID, jobTypes)
// Remove hearbeat
_, err = conn.Do("DEL", redisKeyHeartbeat(r.namespace, deadPoolID))
if err != nil {
lockJobTypes := jobTypes
// if we found jobs from the heartbeat, requeue them and remove the heartbeat
if len(jobTypes) > 0 {
r.requeueInProgressJobs(deadPoolID, jobTypes)
if _, err = conn.Do("DEL", redisKeyHeartbeat(r.namespace, deadPoolID)); err != nil {
return err
}
} else {
// try to clean up locks for the current set of jobs if heartbeat was not found
lockJobTypes = r.curJobTypes
}
// Remove dead pool from worker pools set
if _, err = conn.Do("SREM", workerPoolsKey, deadPoolID); err != nil {
return err
}
// Remove from set
_, err = conn.Do("SREM", workerPoolsKey, deadPoolID)
if err != nil {
// Cleanup any stale lock info
if err = r.cleanStaleLockInfo(deadPoolID, lockJobTypes); err != nil {
return err
}
}
@ -99,13 +108,35 @@ func (r *deadPoolReaper) reap() error {
return nil
}
func (r *deadPoolReaper) requeueInProgressJobs(poolID string, jobTypes []string) error {
redisRequeueScript := redis.NewScript(len(jobTypes)*2, redisLuaRpoplpushMultiCmd)
func (r *deadPoolReaper) cleanStaleLockInfo(poolID string, jobTypes []string) error {
numKeys := len(jobTypes) * 2
redisReapLocksScript := redis.NewScript(numKeys, redisLuaReapStaleLocks)
var scriptArgs = make([]interface{}, 0, numKeys+1) // +1 for argv[1]
var scriptArgs = make([]interface{}, 0, len(jobTypes)*2)
for _, jobType := range jobTypes {
scriptArgs = append(scriptArgs, redisKeyJobsInProgress(r.namespace, poolID, jobType), redisKeyJobs(r.namespace, jobType))
scriptArgs = append(scriptArgs, redisKeyJobsLock(r.namespace, jobType), redisKeyJobsLockInfo(r.namespace, jobType))
}
scriptArgs = append(scriptArgs, poolID) // ARGV[1]
conn := r.pool.Get()
defer conn.Close()
if _, err := redisReapLocksScript.Do(conn, scriptArgs...); err != nil {
return err
}
return nil
}
func (r *deadPoolReaper) requeueInProgressJobs(poolID string, jobTypes []string) error {
numKeys := len(jobTypes) * requeueKeysPerJob
redisRequeueScript := redis.NewScript(numKeys, redisLuaReenqueueJob)
var scriptArgs = make([]interface{}, 0, numKeys+1)
for _, jobType := range jobTypes {
// pops from in progress, push into job queue and decrement the queue lock
scriptArgs = append(scriptArgs, redisKeyJobsInProgress(r.namespace, poolID, jobType), redisKeyJobs(r.namespace, jobType), redisKeyJobsLock(r.namespace, jobType), redisKeyJobsLockInfo(r.namespace, jobType)) // KEYS[1-4 * N]
}
scriptArgs = append(scriptArgs, poolID) // ARGV[1]
conn := r.pool.Get()
defer conn.Close()
@ -139,17 +170,18 @@ func (r *deadPoolReaper) findDeadPools() (map[string][]string, error) {
deadPools := map[string][]string{}
for _, workerPoolID := range workerPoolIDs {
heartbeatKey := redisKeyHeartbeat(r.namespace, workerPoolID)
// Check that last heartbeat was long enough ago to consider the pool dead
heartbeatAt, err := redis.Int64(conn.Do("HGET", heartbeatKey, "heartbeat_at"))
if err == redis.ErrNil {
// heartbeat expired, save dead pool and use cur set of jobs from reaper
deadPools[workerPoolID] = []string{}
continue
}
if err != nil {
return nil, err
}
if time.Unix(heartbeatAt, 0).Add(deadTime).After(time.Now()) {
// Check that last heartbeat was long enough ago to consider the pool dead
if time.Unix(heartbeatAt, 0).Add(r.deadTime).After(time.Now()) {
continue
}

View File

@ -1,15 +1,17 @@
package work
import (
"github.com/garyburd/redigo/redis"
"github.com/stretchr/testify/assert"
"testing"
"time"
"github.com/gomodule/redigo/redis"
"github.com/stretchr/testify/assert"
)
func TestDeadPoolReaper(t *testing.T) {
pool := newTestPool(":6379")
ns := "work"
cleanKeyspace(ns, pool)
conn := pool.Get()
defer conn.Close()
@ -18,7 +20,6 @@ func TestDeadPoolReaper(t *testing.T) {
// Create redis data
var err error
cleanKeyspace(ns, pool)
err = conn.Send("SADD", workerPoolsKey, "1")
assert.NoError(t, err)
err = conn.Send("SADD", workerPoolsKey, "2")
@ -47,14 +48,18 @@ func TestDeadPoolReaper(t *testing.T) {
assert.NoError(t, err)
// Test getting dead pool
reaper := newDeadPoolReaper(ns, pool)
reaper := newDeadPoolReaper(ns, pool, []string{})
deadPools, err := reaper.findDeadPools()
assert.NoError(t, err)
assert.Equal(t, deadPools, map[string][]string{"2": {"type1", "type2"}, "3": {"type1", "type2"}})
assert.Equal(t, map[string][]string{"2": {"type1", "type2"}, "3": {"type1", "type2"}}, deadPools)
// Test requeueing jobs
_, err = conn.Do("lpush", redisKeyJobsInProgress(ns, "2", "type1"), "foo")
assert.NoError(t, err)
_, err = conn.Do("incr", redisKeyJobsLock(ns, "type1"))
assert.NoError(t, err)
_, err = conn.Do("hincrby", redisKeyJobsLockInfo(ns, "type1"), "2", 1) // worker pool 2 has lock
assert.NoError(t, err)
// Ensure 0 jobs in jobs queue
jobsCount, err := redis.Int(conn.Do("llen", redisKeyJobs(ns, "type1")))
@ -79,6 +84,11 @@ func TestDeadPoolReaper(t *testing.T) {
jobsCount, err = redis.Int(conn.Do("llen", redisKeyJobsInProgress(ns, "2", "type1")))
assert.NoError(t, err)
assert.Equal(t, 0, jobsCount)
// Locks should get cleaned up
assert.EqualValues(t, 0, getInt64(pool, redisKeyJobsLock(ns, "type1")))
v, _ := conn.Do("HGET", redisKeyJobsLockInfo(ns, "type1"), "2")
assert.Nil(t, v)
}
func TestDeadPoolReaperNoHeartbeat(t *testing.T) {
@ -99,14 +109,28 @@ func TestDeadPoolReaperNoHeartbeat(t *testing.T) {
assert.NoError(t, err)
err = conn.Send("SADD", workerPoolsKey, "3")
assert.NoError(t, err)
// stale lock info
err = conn.Send("SET", redisKeyJobsLock(ns, "type1"), 3)
assert.NoError(t, err)
err = conn.Send("HSET", redisKeyJobsLockInfo(ns, "type1"), "1", 1)
assert.NoError(t, err)
err = conn.Send("HSET", redisKeyJobsLockInfo(ns, "type1"), "2", 1)
assert.NoError(t, err)
err = conn.Send("HSET", redisKeyJobsLockInfo(ns, "type1"), "3", 1)
assert.NoError(t, err)
err = conn.Flush()
assert.NoError(t, err)
// Test getting dead pool
reaper := newDeadPoolReaper(ns, pool)
// make sure test data was created
numPools, err := redis.Int(conn.Do("scard", workerPoolsKey))
assert.NoError(t, err)
assert.EqualValues(t, 3, numPools)
// Test getting dead pool ids
reaper := newDeadPoolReaper(ns, pool, []string{"type1"})
deadPools, err := reaper.findDeadPools()
assert.NoError(t, err)
assert.Equal(t, deadPools, map[string][]string{})
assert.Equal(t, map[string][]string{"1": {}, "2": {}, "3": {}}, deadPools)
// Test requeueing jobs
_, err = conn.Do("lpush", redisKeyJobsInProgress(ns, "2", "type1"), "foo")
@ -122,24 +146,42 @@ func TestDeadPoolReaperNoHeartbeat(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 1, jobsCount)
// Ensure dead worker pools still in the set
jobsCount, err = redis.Int(conn.Do("scard", redisKeyWorkerPools(ns)))
assert.NoError(t, err)
assert.Equal(t, 3, jobsCount)
// Reap
err = reaper.reap()
assert.NoError(t, err)
// Ensure 0 jobs in jobs queue
// Ensure jobs queue was not altered
jobsCount, err = redis.Int(conn.Do("llen", redisKeyJobs(ns, "type1")))
assert.NoError(t, err)
assert.Equal(t, 0, jobsCount)
// Ensure 1 job in inprogress queue
// Ensure inprogress queue was not altered
jobsCount, err = redis.Int(conn.Do("llen", redisKeyJobsInProgress(ns, "2", "type1")))
assert.NoError(t, err)
assert.Equal(t, 1, jobsCount)
// Ensure dead worker pools were removed from the set
jobsCount, err = redis.Int(conn.Do("scard", redisKeyWorkerPools(ns)))
assert.NoError(t, err)
assert.Equal(t, 0, jobsCount)
// Stale lock info was cleaned up using reap.curJobTypes
assert.EqualValues(t, 0, getInt64(pool, redisKeyJobsLock(ns, "type1")))
for _, poolID := range []string{"1", "2", "3"} {
v, _ := conn.Do("HGET", redisKeyJobsLockInfo(ns, "type1"), poolID)
assert.Nil(t, v)
}
}
func TestDeadPoolReaperNoJobTypes(t *testing.T) {
pool := newTestPool(":6379")
ns := "work"
cleanKeyspace(ns, pool)
conn := pool.Get()
defer conn.Close()
@ -148,7 +190,6 @@ func TestDeadPoolReaperNoJobTypes(t *testing.T) {
// Create redis data
var err error
cleanKeyspace(ns, pool)
err = conn.Send("SADD", workerPoolsKey, "1")
assert.NoError(t, err)
err = conn.Send("SADD", workerPoolsKey, "2")
@ -169,10 +210,10 @@ func TestDeadPoolReaperNoJobTypes(t *testing.T) {
assert.NoError(t, err)
// Test getting dead pool
reaper := newDeadPoolReaper(ns, pool)
reaper := newDeadPoolReaper(ns, pool, []string{})
deadPools, err := reaper.findDeadPools()
assert.NoError(t, err)
assert.Equal(t, deadPools, map[string][]string{"2": {"type1", "type2"}})
assert.Equal(t, map[string][]string{"2": {"type1", "type2"}}, deadPools)
// Test requeueing jobs
_, err = conn.Do("lpush", redisKeyJobsInProgress(ns, "1", "type1"), "foo")
@ -212,3 +253,95 @@ func TestDeadPoolReaperNoJobTypes(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 0, jobsCount)
}
func TestDeadPoolReaperWithWorkerPools(t *testing.T) {
pool := newTestPool(":6379")
ns := "work"
job1 := "job1"
stalePoolID := "aaa"
cleanKeyspace(ns, pool)
// test vars
expectedDeadTime := 5 * time.Millisecond
// create a stale job with a heartbeat
conn := pool.Get()
defer conn.Close()
_, err := conn.Do("SADD", redisKeyWorkerPools(ns), stalePoolID)
assert.NoError(t, err)
_, err = conn.Do("LPUSH", redisKeyJobsInProgress(ns, stalePoolID, job1), `{"sleep": 10}`)
assert.NoError(t, err)
jobTypes := map[string]*jobType{"job1": nil}
staleHeart := newWorkerPoolHeartbeater(ns, pool, stalePoolID, jobTypes, 1, []string{"id1"})
staleHeart.start()
// should have 1 stale job and empty job queue
assert.EqualValues(t, 1, listSize(pool, redisKeyJobsInProgress(ns, stalePoolID, job1)))
assert.EqualValues(t, 0, listSize(pool, redisKeyJobs(ns, job1)))
// setup a worker pool and start the reaper, which should restart the stale job above
wp := setupTestWorkerPool(pool, ns, job1, 1, JobOptions{Priority: 1})
wp.deadPoolReaper = newDeadPoolReaper(wp.namespace, wp.pool, []string{"job1"})
wp.deadPoolReaper.deadTime = expectedDeadTime
wp.deadPoolReaper.start()
// sleep long enough for staleJob to be considered dead
time.Sleep(expectedDeadTime * 2)
// now we should have 1 job in queue and no more stale jobs
assert.EqualValues(t, 1, listSize(pool, redisKeyJobs(ns, job1)))
assert.EqualValues(t, 0, listSize(pool, redisKeyJobsInProgress(ns, wp.workerPoolID, job1)))
staleHeart.stop()
wp.deadPoolReaper.stop()
}
func TestDeadPoolReaperCleanStaleLocks(t *testing.T) {
pool := newTestPool(":6379")
ns := "work"
cleanKeyspace(ns, pool)
conn := pool.Get()
defer conn.Close()
job1, job2 := "type1", "type2"
jobNames := []string{job1, job2}
workerPoolID1, workerPoolID2 := "1", "2"
lock1 := redisKeyJobsLock(ns, job1)
lock2 := redisKeyJobsLock(ns, job2)
lockInfo1 := redisKeyJobsLockInfo(ns, job1)
lockInfo2 := redisKeyJobsLockInfo(ns, job2)
// Create redis data
var err error
err = conn.Send("SET", lock1, 3)
assert.NoError(t, err)
err = conn.Send("SET", lock2, 1)
assert.NoError(t, err)
err = conn.Send("HSET", lockInfo1, workerPoolID1, 1) // workerPoolID1 holds 1 lock on job1
assert.NoError(t, err)
err = conn.Send("HSET", lockInfo1, workerPoolID2, 2) // workerPoolID2 holds 2 locks on job1
assert.NoError(t, err)
err = conn.Send("HSET", lockInfo2, workerPoolID2, 2) // test that we don't go below 0 on job2 lock
assert.NoError(t, err)
err = conn.Flush()
assert.NoError(t, err)
reaper := newDeadPoolReaper(ns, pool, jobNames)
// clean lock info for workerPoolID1
reaper.cleanStaleLockInfo(workerPoolID1, jobNames)
assert.NoError(t, err)
assert.EqualValues(t, 2, getInt64(pool, lock1)) // job1 lock should be decr by 1
assert.EqualValues(t, 1, getInt64(pool, lock2)) // job2 lock is unchanged
v, _ := conn.Do("HGET", lockInfo1, workerPoolID1) // workerPoolID1 removed from job1's lock info
assert.Nil(t, v)
// now clean lock info for workerPoolID2
reaper.cleanStaleLockInfo(workerPoolID2, jobNames)
assert.NoError(t, err)
// both locks should be at 0
assert.EqualValues(t, 0, getInt64(pool, lock1))
assert.EqualValues(t, 0, getInt64(pool, lock2))
// worker pool ID 2 removed from both lock info hashes
v, err = conn.Do("HGET", lockInfo1, workerPoolID2)
assert.Nil(t, v)
v, err = conn.Do("HGET", lockInfo2, workerPoolID2)
assert.Nil(t, v)
}

View File

@ -1,8 +1,10 @@
package work
import (
"github.com/garyburd/redigo/redis"
"sync"
"time"
"github.com/gomodule/redigo/redis"
)
// Enqueuer can enqueue jobs.
@ -14,6 +16,7 @@ type Enqueuer struct {
knownJobs map[string]int64
enqueueUniqueScript *redis.Script
enqueueUniqueInScript *redis.Script
mtx sync.RWMutex
}
// NewEnqueuer creates a new enqueuer with the specified Redis namespace and Redis pool.
@ -95,7 +98,10 @@ func (e *Enqueuer) EnqueueIn(jobName string, secondsFromNow int64, args map[stri
return scheduledJob, nil
}
// EnqueueUnique enqueues a job unless a job is already enqueued with the same name and arguments. The already-enqueued job can be in the normal work queue or in the scheduled job queue. Once a worker begins processing a job, another job with the same name and arguments can be enqueued again. Any failed jobs in the retry queue or dead queue don't count against the uniqueness -- so if a job fails and is retried, two unique jobs with the same name and arguments can be enqueued at once.
// EnqueueUnique enqueues a job unless a job is already enqueued with the same name and arguments.
// The already-enqueued job can be in the normal work queue or in the scheduled job queue.
// Once a worker begins processing a job, another job with the same name and arguments can be enqueued again.
// Any failed jobs in the retry queue or dead queue don't count against the uniqueness -- so if a job fails and is retried, two unique jobs with the same name and arguments can be enqueued at once.
// In order to add robustness to the system, jobs are only unique for 24 hours after they're enqueued. This is mostly relevant for scheduled jobs.
// EnqueueUnique returns the job if it was enqueued and nil if it wasn't
func (e *Enqueuer) EnqueueUnique(jobName string, args map[string]interface{}) (*Job, error) {
@ -185,7 +191,11 @@ func (e *Enqueuer) EnqueueUniqueIn(jobName string, secondsFromNow int64, args ma
func (e *Enqueuer) addToKnownJobs(conn redis.Conn, jobName string) error {
needSadd := true
now := time.Now().Unix()
e.mtx.RLock()
t, ok := e.knownJobs[jobName]
e.mtx.RUnlock()
if ok {
if now < t {
needSadd = false
@ -195,7 +205,10 @@ func (e *Enqueuer) addToKnownJobs(conn redis.Conn, jobName string) error {
if _, err := conn.Do("SADD", redisKeyKnownJobs(e.Namespace), jobName); err != nil {
return err
}
e.mtx.Lock()
e.knownJobs[jobName] = now + 300
e.mtx.Unlock()
}
return nil

View File

@ -2,9 +2,11 @@ package work
import (
"fmt"
"github.com/stretchr/testify/assert"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
)
func TestEnqueue(t *testing.T) {
@ -101,7 +103,7 @@ func TestEnqueueUnique(t *testing.T) {
ns := "work"
cleanKeyspace(ns, pool)
enqueuer := NewEnqueuer(ns, pool)
var mutex = &sync.Mutex{}
job, err := enqueuer.EnqueueUnique("wat", Q{"a": 1, "b": "cool"})
assert.NoError(t, err)
if assert.NotNil(t, job) {
@ -134,15 +136,19 @@ func TestEnqueueUnique(t *testing.T) {
assert.NoError(t, err)
assert.NotNil(t, job)
// Process the queues. Ensure the right numbero of jobs was processed
// Process the queues. Ensure the right number of jobs were processed
var wats, taws int64
wp := NewWorkerPool(TestContext{}, 3, ns, pool)
wp.JobWithOptions("wat", JobOptions{Priority: 1, MaxFails: 1}, func(job *Job) error {
mutex.Lock()
wats++
mutex.Unlock()
return nil
})
wp.JobWithOptions("taw", JobOptions{Priority: 1, MaxFails: 1}, func(job *Job) error {
mutex.Lock()
taws++
mutex.Unlock()
return fmt.Errorf("ohno")
})
wp.Start()

View File

@ -1,26 +1,29 @@
package work
import (
// "fmt"
"github.com/garyburd/redigo/redis"
"os"
"sort"
"strings"
"time"
"github.com/gomodule/redigo/redis"
)
const (
beatPeriod = 5 * time.Second
)
type workerPoolHeartbeater struct {
workerPoolID string
namespace string // eg, "myapp-work"
pool *redis.Pool
//
concurrency uint
jobNames string
startedAt int64
pid int
hostname string
workerIDs string
beatPeriod time.Duration
concurrency uint
jobNames string
startedAt int64
pid int
hostname string
workerIDs string
stopChan chan struct{}
doneStoppingChan chan struct{}
@ -28,12 +31,11 @@ type workerPoolHeartbeater struct {
func newWorkerPoolHeartbeater(namespace string, pool *redis.Pool, workerPoolID string, jobTypes map[string]*jobType, concurrency uint, workerIDs []string) *workerPoolHeartbeater {
h := &workerPoolHeartbeater{
workerPoolID: workerPoolID,
namespace: namespace,
pool: pool,
concurrency: concurrency,
workerPoolID: workerPoolID,
namespace: namespace,
pool: pool,
beatPeriod: beatPeriod,
concurrency: concurrency,
stopChan: make(chan struct{}),
doneStoppingChan: make(chan struct{}),
}
@ -71,7 +73,7 @@ func (h *workerPoolHeartbeater) stop() {
func (h *workerPoolHeartbeater) loop() {
h.startedAt = nowEpochSeconds()
h.heartbeat() // do it right away
ticker := time.Tick(5000 * time.Millisecond)
ticker := time.Tick(h.beatPeriod)
for {
select {
case <-h.stopChan:
@ -101,7 +103,6 @@ func (h *workerPoolHeartbeater) heartbeat() {
"host", h.hostname,
"pid", h.pid,
)
conn.Send("EXPIRE", heartbeatKey, 60)
if err := conn.Flush(); err != nil {
logError("heartbeat", err)

View File

@ -1,11 +1,11 @@
package work
import (
// "fmt"
"github.com/garyburd/redigo/redis"
"github.com/stretchr/testify/assert"
"testing"
"time"
"github.com/gomodule/redigo/redis"
"github.com/stretchr/testify/assert"
)
func TestHeartbeater(t *testing.T) {

View File

@ -1,8 +1,6 @@
package work
import (
"testing"
)
import "testing"
func TestMakeIdentifier(t *testing.T) {
id := makeIdentifier()

View File

@ -1,9 +1,10 @@
package work
import (
"github.com/stretchr/testify/assert"
"math"
"testing"
"github.com/stretchr/testify/assert"
)
func TestJobArgumentExtraction(t *testing.T) {

View File

@ -1,8 +1,6 @@
package work
import (
"fmt"
)
import "fmt"
func logError(key string, err error) {
fmt.Printf("ERROR: %s - %s\n", key, err.Error())

View File

@ -3,8 +3,9 @@ package work
import (
"encoding/json"
"fmt"
"github.com/garyburd/redigo/redis"
"time"
"github.com/gomodule/redigo/redis"
)
// An observer observes a single worker. Each worker has its own observer.
@ -120,7 +121,7 @@ func (o *observer) observeCheckin(jobName, jobID, checkin string) {
}
func (o *observer) loop() {
// Ever tick, we'll update redis if necessary
// Every tick we'll update redis if necessary
// We don't update it on every job because the only purpose of this data is for humans to inspect the system,
// and a fast worker could move onto new jobs every few ms.
ticker := time.Tick(1000 * time.Millisecond)

View File

@ -2,10 +2,10 @@ package work
import (
"fmt"
"github.com/garyburd/redigo/redis"
"github.com/stretchr/testify/assert"
"testing"
// "time"
"github.com/gomodule/redigo/redis"
"github.com/stretchr/testify/assert"
)
func TestObserverStarted(t *testing.T) {

View File

@ -2,10 +2,11 @@ package work
import (
"fmt"
"github.com/garyburd/redigo/redis"
"github.com/robfig/cron"
"math/rand"
"time"
"github.com/gomodule/redigo/redis"
"github.com/robfig/cron"
)
const (

View File

@ -1,11 +1,12 @@
package work
import (
"github.com/garyburd/redigo/redis"
"github.com/robfig/cron"
"github.com/stretchr/testify/assert"
"testing"
"time"
"github.com/gomodule/redigo/redis"
"github.com/robfig/cron"
"github.com/stretchr/testify/assert"
)
func TestPeriodicEnqueuer(t *testing.T) {

View File

@ -13,15 +13,23 @@ type sampleItem struct {
priority uint
// payload:
redisJobs string
redisJobsInProg string
redisJobs string
redisJobsInProg string
redisJobsPaused string
redisJobsLock string
redisJobsLockInfo string
redisJobsMaxConcurrency string
}
func (s *prioritySampler) add(priority uint, redisJobs, redisJobsInProg string) {
func (s *prioritySampler) add(priority uint, redisJobs, redisJobsInProg, redisJobsPaused, redisJobsLock, redisJobsLockInfo, redisJobsMaxConcurrency string) {
sample := sampleItem{
priority: priority,
redisJobs: redisJobs,
redisJobsInProg: redisJobsInProg,
priority: priority,
redisJobs: redisJobs,
redisJobsInProg: redisJobsInProg,
redisJobsPaused: redisJobsPaused,
redisJobsLock: redisJobsLock,
redisJobsLockInfo: redisJobsLockInfo,
redisJobsMaxConcurrency: redisJobsMaxConcurrency,
}
s.samples = append(s.samples, sample)
s.sum += priority

View File

@ -2,15 +2,17 @@ package work
import (
"fmt"
"github.com/stretchr/testify/assert"
"testing"
"github.com/stretchr/testify/assert"
)
func TestPrioritySampler(t *testing.T) {
ps := prioritySampler{}
ps.add(5, "jobs.5", "jobsinprog.5")
ps.add(2, "jobs.2a", "jobsinprog.2a")
ps.add(1, "jobs.1b", "jobsinprog.1b")
ps.add(5, "jobs.5", "jobsinprog.5", "jobspaused.5", "jobslock.5", "jobslockinfo.5", "jobsconcurrency.5")
ps.add(2, "jobs.2a", "jobsinprog.2a", "jobspaused.2a", "jobslock.2a", "jobslockinfo.2a", "jobsconcurrency.2a")
ps.add(1, "jobs.1b", "jobsinprog.1b", "jobspaused.1b", "jobslock.1b", "jobslockinfo.1b", "jobsconcurrency.1b")
var c5 = 0
var c2 = 0
@ -41,7 +43,13 @@ func TestPrioritySampler(t *testing.T) {
func BenchmarkPrioritySampler(b *testing.B) {
ps := prioritySampler{}
for i := 0; i < 200; i++ {
ps.add(uint(i)+1, "jobs."+fmt.Sprint(i), "jobsinprog."+fmt.Sprint(i))
ps.add(uint(i)+1,
"jobs."+fmt.Sprint(i),
"jobsinprog."+fmt.Sprint(i),
"jobspaused."+fmt.Sprint(i),
"jobslock."+fmt.Sprint(i),
"jobslockinfo."+fmt.Sprint(i),
"jobsmaxconcurrency."+fmt.Sprint(i))
}
b.ResetTimer()

View File

@ -56,6 +56,22 @@ func redisKeyHeartbeat(namespace, workerPoolID string) string {
return redisNamespacePrefix(namespace) + "worker_pools:" + workerPoolID
}
func redisKeyJobsPaused(namespace, jobName string) string {
return redisKeyJobs(namespace, jobName) + ":paused"
}
func redisKeyJobsLock(namespace, jobName string) string {
return redisKeyJobs(namespace, jobName) + ":lock"
}
func redisKeyJobsLockInfo(namespace, jobName string) string {
return redisKeyJobs(namespace, jobName) + ":lock_info"
}
func redisKeyJobsConcurrency(namespace, jobName string) string {
return redisKeyJobs(namespace, jobName) + ":max_concurrency"
}
func redisKeyUniqueJob(namespace, jobName string, args map[string]interface{}) (string, error) {
var buf bytes.Buffer
@ -78,6 +94,8 @@ func redisKeyLastPeriodicEnqueue(namespace string) string {
return redisNamespacePrefix(namespace) + "last_periodic_enqueue"
}
// Used to fetch the next job to run
//
// KEYS[1] = the 1st job queue we want to try, eg, "work:jobs:emails"
// KEYS[2] = the 1st job queue's in prog queue, eg, "work:jobs:emails:97c84119d13cb54119a38743:inprogress"
// KEYS[3] = the 2nd job queue...
@ -85,13 +103,116 @@ func redisKeyLastPeriodicEnqueue(namespace string) string {
// ...
// KEYS[N] = the last job queue...
// KEYS[N+1] = the last job queue's in prog queue...
var redisLuaRpoplpushMultiCmd = `
local res
// ARGV[1] = job queue's workerPoolID
var redisLuaFetchJob = fmt.Sprintf(`
local function acquireLock(lockKey, lockInfoKey, workerPoolID)
redis.call('incr', lockKey)
redis.call('hincrby', lockInfoKey, workerPoolID, 1)
end
local function haveJobs(jobQueue)
return redis.call('llen', jobQueue) > 0
end
local function isPaused(pauseKey)
return redis.call('get', pauseKey)
end
local function canRun(lockKey, maxConcurrency)
local activeJobs = tonumber(redis.call('get', lockKey))
if (not maxConcurrency or maxConcurrency == 0) or (not activeJobs or activeJobs < maxConcurrency) then
-- default case: maxConcurrency not defined or set to 0 means no cap on concurrent jobs OR
-- maxConcurrency set, but lock does not yet exist OR
-- maxConcurrency set, lock is set, but not yet at max concurrency
return true
else
-- we are at max capacity for running jobs
return false
end
end
local res, jobQueue, inProgQueue, pauseKey, lockKey, maxConcurrency, workerPoolID, concurrencyKey, lockInfoKey
local keylen = #KEYS
for i=1,keylen,2 do
res = redis.call('rpoplpush', KEYS[i], KEYS[i+1])
workerPoolID = ARGV[1]
for i=1,keylen,%d do
jobQueue = KEYS[i]
inProgQueue = KEYS[i+1]
pauseKey = KEYS[i+2]
lockKey = KEYS[i+3]
lockInfoKey = KEYS[i+4]
concurrencyKey = KEYS[i+5]
maxConcurrency = tonumber(redis.call('get', concurrencyKey))
if haveJobs(jobQueue) and not isPaused(pauseKey) and canRun(lockKey, maxConcurrency) then
acquireLock(lockKey, lockInfoKey, workerPoolID)
res = redis.call('rpoplpush', jobQueue, inProgQueue)
return {res, jobQueue, inProgQueue}
end
end
return nil`, fetchKeysPerJobType)
// Used by the reaper to re-enqueue jobs that were in progress
//
// KEYS[1] = the 1st job's in progress queue
// KEYS[2] = the 1st job's job queue
// KEYS[3] = the 2nd job's in progress queue
// KEYS[4] = the 2nd job's job queue
// ...
// KEYS[N] = the last job's in progress queue
// KEYS[N+1] = the last job's job queue
// ARGV[1] = workerPoolID for job queue
var redisLuaReenqueueJob = fmt.Sprintf(`
local function releaseLock(lockKey, lockInfoKey, workerPoolID)
redis.call('decr', lockKey)
redis.call('hincrby', lockInfoKey, workerPoolID, -1)
end
local keylen = #KEYS
local res, jobQueue, inProgQueue, workerPoolID, lockKey, lockInfoKey
workerPoolID = ARGV[1]
for i=1,keylen,%d do
inProgQueue = KEYS[i]
jobQueue = KEYS[i+1]
lockKey = KEYS[i+2]
lockInfoKey = KEYS[i+3]
res = redis.call('rpoplpush', inProgQueue, jobQueue)
if res then
return {res, KEYS[i], KEYS[i+1]}
releaseLock(lockKey, lockInfoKey, workerPoolID)
return {res, inProgQueue, jobQueue}
end
end
return nil`, requeueKeysPerJob)
// Used by the reaper to clean up stale locks
//
// KEYS[1] = the 1st job's lock
// KEYS[2] = the 1st job's lock info hash
// KEYS[3] = the 2nd job's lock
// KEYS[4] = the 2nd job's lock info hash
// ...
// KEYS[N] = the last job's lock
// KEYS[N+1] = the last job's lock info haash
// ARGV[1] = the dead worker pool id
var redisLuaReapStaleLocks = `
local keylen = #KEYS
local lock, lockInfo, deadLockCount
local deadPoolID = ARGV[1]
for i=1,keylen,2 do
lock = KEYS[i]
lockInfo = KEYS[i+1]
deadLockCount = tonumber(redis.call('hget', lockInfo, deadPoolID))
if deadLockCount then
redis.call('decrby', lock, deadLockCount)
redis.call('hdel', lockInfo, deadPoolID)
if tonumber(redis.call('get', lock)) < 0 then
redis.call('set', lock, 0)
end
end
end
return nil
@ -140,8 +261,8 @@ for i=1,jobCount do
j = cjson.decode(jobs[i])
if j['id'] == ARGV[2] then
redis.call('zrem', KEYS[1], jobs[i])
deletedCount = deletedCount + 1
jobBytes = jobs[i]
deletedCount = deletedCount + 1
jobBytes = jobs[i]
end
end
return {deletedCount, jobBytes}
@ -225,7 +346,7 @@ return requeuedCount
`
// KEYS[1] = job queue to push onto
// KEYS[2] = Unique job's key. Test for existance and set if we push.
// KEYS[2] = Unique job's key. Test for existence and set if we push.
// ARGV[1] = job
var redisLuaEnqueueUnique = `
if redis.call('set', KEYS[2], '1', 'NX', 'EX', '86400') then
@ -236,7 +357,7 @@ return 'dup'
`
// KEYS[1] = scheduled job queue
// KEYS[2] = Unique job's key. Test for existance and set if we push.
// KEYS[2] = Unique job's key. Test for existence and set if we push.
// ARGV[1] = job
// ARGV[2] = epoch seconds for job to be run at
var redisLuaEnqueueUniqueIn = `

View File

@ -1,11 +1,10 @@
package work
import (
// "encoding/json"
"fmt"
"time"
"github.com/garyburd/redigo/redis"
"github.com/gomodule/redigo/redis"
)
type requeuer struct {

View File

@ -1,12 +1,9 @@
package work
import (
// "github.com/garyburd/redigo/redis"
"github.com/stretchr/testify/assert"
"testing"
// "fmt"
// "time"
// "os"
"github.com/stretchr/testify/assert"
)
func TestRequeue(t *testing.T) {

View File

@ -2,9 +2,10 @@ package work
import (
"fmt"
"github.com/stretchr/testify/assert"
"reflect"
"testing"
"github.com/stretchr/testify/assert"
)
func TestRunBasicMiddleware(t *testing.T) {

View File

@ -1,8 +1,6 @@
package work
import (
"time"
)
import "time"
var nowMock int64

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -15,17 +15,22 @@
"babel-preset-react": "latest",
"babel-preset-stage-0": "latest",
"css-loader": "latest",
"enzyme": "^3.2.0",
"enzyme-adapter-react-16": "^1.1.0",
"eslint": "latest",
"eslint-plugin-react": "latest",
"expect": "latest",
"file-loader": "latest",
"ignore-styles": "latest",
"istanbul": "^1.1.0-alpha.1",
"jsdom": "^11.5.1",
"jsdom-global": "^3.0.2",
"mocha": "latest",
"react": "latest",
"react-addons-test-utils": "latest",
"react-dom": "latest",
"react-router": "latest",
"react-router": "3.2.0",
"react-shallow-renderer-helpers": "^2.0.2",
"style-loader": "latest",
"url-loader": "latest",
"webpack": "latest"

View File

@ -1,4 +1,5 @@
import React from 'react';
import PropTypes from 'prop-types';
import PageList from './PageList';
import UnixTime from './UnixTime';
import styles from './bootstrap.min.css';
@ -6,11 +7,11 @@ import cx from './cx';
export default class DeadJobs extends React.Component {
static propTypes = {
fetchURL: React.PropTypes.string,
deleteURL: React.PropTypes.string,
deleteAllURL: React.PropTypes.string,
retryURL: React.PropTypes.string,
retryAllURL: React.PropTypes.string,
fetchURL: PropTypes.string,
deleteURL: PropTypes.string,
deleteAllURL: PropTypes.string,
retryURL: PropTypes.string,
retryAllURL: PropTypes.string,
}
state = {
@ -147,7 +148,7 @@ export default class DeadJobs extends React.Component {
<td>{job.err}</td>
<td><UnixTime ts={job.t} /></td>
</tr>
);
);
})
}
</tbody>

View File

@ -1,17 +1,15 @@
import './TestSetup';
import expect from 'expect';
import DeadJobs from './DeadJobs';
import React from 'react';
import ReactTestUtils from 'react-addons-test-utils';
import { findAllByTag } from './TestUtils';
import { mount } from 'enzyme';
describe('DeadJobs', () => {
it('shows dead jobs', () => {
let r = ReactTestUtils.createRenderer();
r.render(<DeadJobs />);
let deadJobs = r.getMountedInstance();
let deadJobs = mount(<DeadJobs />);
expect(deadJobs.state.selected.length).toEqual(0);
expect(deadJobs.state.jobs.length).toEqual(0);
expect(deadJobs.state().selected.length).toEqual(0);
expect(deadJobs.state().jobs.length).toEqual(0);
deadJobs.setState({
count: 2,
@ -21,64 +19,54 @@ describe('DeadJobs', () => {
]
});
expect(deadJobs.state.selected.length).toEqual(0);
expect(deadJobs.state.jobs.length).toEqual(2);
expect(deadJobs.state().selected.length).toEqual(0);
expect(deadJobs.state().jobs.length).toEqual(2);
let output = r.getRenderOutput();
let checkbox = findAllByTag(output, 'input');
let checkbox = deadJobs.find('input');
expect(checkbox.length).toEqual(3);
expect(checkbox.at(0).props().checked).toEqual(false);
expect(checkbox.at(1).props().checked).toEqual(false);
expect(checkbox.at(2).props().checked).toEqual(false);
expect(checkbox[0].props.checked).toEqual(false);
expect(checkbox[1].props.checked).toEqual(false);
expect(checkbox[2].props.checked).toEqual(false);
checkbox[0].props.onChange();
output = r.getRenderOutput();
checkbox = findAllByTag(output, 'input');
checkbox.at(0).simulate('change');
checkbox = deadJobs.find('input');
expect(checkbox.length).toEqual(3);
expect(checkbox[0].props.checked).toEqual(true);
expect(checkbox[1].props.checked).toEqual(true);
expect(checkbox[2].props.checked).toEqual(true);
expect(checkbox.at(0).props().checked).toEqual(true);
expect(checkbox.at(1).props().checked).toEqual(true);
expect(checkbox.at(2).props().checked).toEqual(true);
checkbox[1].props.onChange();
output = r.getRenderOutput();
checkbox = findAllByTag(output, 'input');
checkbox.at(1).simulate('change');
checkbox = deadJobs.find('input');
expect(checkbox.length).toEqual(3);
expect(checkbox[0].props.checked).toEqual(true);
expect(checkbox[1].props.checked).toEqual(false);
expect(checkbox[2].props.checked).toEqual(true);
expect(checkbox.at(0).props().checked).toEqual(true);
expect(checkbox.at(1).props().checked).toEqual(false);
expect(checkbox.at(2).props().checked).toEqual(true);
checkbox[1].props.onChange();
output = r.getRenderOutput();
checkbox = findAllByTag(output, 'input');
checkbox.at(1).simulate('change');
checkbox = deadJobs.find('input');
expect(checkbox.length).toEqual(3);
expect(checkbox[0].props.checked).toEqual(true);
expect(checkbox[1].props.checked).toEqual(true);
expect(checkbox[2].props.checked).toEqual(true);
expect(checkbox.at(0).props().checked).toEqual(true);
expect(checkbox.at(1).props().checked).toEqual(true);
expect(checkbox.at(2).props().checked).toEqual(true);
let button = findAllByTag(output, 'button');
let button = deadJobs.find('button');
expect(button.length).toEqual(4);
button[0].props.onClick();
button[1].props.onClick();
button[2].props.onClick();
button[3].props.onClick();
button.at(0).simulate('click');
button.at(1).simulate('click');
button.at(2).simulate('click');
button.at(3).simulate('click');
checkbox[0].props.onChange();
checkbox.at(0).simulate('change');
output = r.getRenderOutput();
checkbox = findAllByTag(output, 'input');
checkbox = deadJobs.find('input');
expect(checkbox.length).toEqual(3);
expect(checkbox[0].props.checked).toEqual(false);
expect(checkbox[1].props.checked).toEqual(false);
expect(checkbox[2].props.checked).toEqual(false);
expect(checkbox.at(0).props().checked).toEqual(false);
expect(checkbox.at(1).props().checked).toEqual(false);
expect(checkbox.at(2).props().checked).toEqual(false);
});
it('has pages', () => {
let r = ReactTestUtils.createRenderer();
r.render(<DeadJobs />);
let deadJobs = r.getMountedInstance();
let deadJobs = mount(<DeadJobs />);
let genJob = (n) => {
let job = [];
@ -98,14 +86,13 @@ describe('DeadJobs', () => {
jobs: genJob(21)
});
expect(deadJobs.state.jobs.length).toEqual(21);
expect(deadJobs.state.page).toEqual(1);
expect(deadJobs.state().jobs.length).toEqual(21);
expect(deadJobs.state().page).toEqual(1);
let output = r.getRenderOutput();
let pageList = findAllByTag(output, 'PageList');
let pageList = deadJobs.find('PageList');
expect(pageList.length).toEqual(1);
pageList[0].props.jumpTo(2)();
expect(deadJobs.state.page).toEqual(2);
pageList.at(0).props().jumpTo(2)();
expect(deadJobs.state().page).toEqual(2);
});
});

View File

@ -1,12 +1,13 @@
import React from 'react';
import PropTypes from 'prop-types';
import styles from './bootstrap.min.css';
export default class PageList extends React.Component {
static propTypes = {
page: React.PropTypes.number.isRequired,
perPage: React.PropTypes.number.isRequired,
totalCount: React.PropTypes.number.isRequired,
jumpTo: React.PropTypes.func.isRequired,
page: PropTypes.number.isRequired,
perPage: PropTypes.number.isRequired,
totalCount: PropTypes.number.isRequired,
jumpTo: PropTypes.func.isRequired,
}
get totalPage() {

View File

@ -1,16 +1,16 @@
import './TestSetup';
import expect from 'expect';
import PageList from './PageList';
import React from 'react';
import ReactTestUtils from 'react-addons-test-utils';
import { mount } from 'enzyme';
describe('PageList', () => {
it('lists pages', () => {
let assertPage = (n, expected) => {
let r = ReactTestUtils.createRenderer();
r.render(<PageList page={n} perPage={2} totalCount={13} jumpTo={() => () => {}} />);
let output = r.getRenderOutput();
expect(output.type).toEqual('ul');
expect(output.props.children.map((el) => {
let pageList = mount(<PageList page={n} perPage={2} totalCount={13} jumpTo={() => () => {}} />);
let ul = pageList.find('ul');
expect(ul.props().children.map((el) => {
expect(el.type).toEqual('li');
return el.props.children.props.children;
})).toEqual(expected);
@ -26,10 +26,8 @@ describe('PageList', () => {
});
it('renders nothing if there is nothing', () => {
let r = ReactTestUtils.createRenderer();
r.render(<PageList page={1} perPage={2} totalCount={0} jumpTo={() => () => {}} />);
let output = r.getRenderOutput();
let pageList = mount(<PageList page={1} perPage={2} totalCount={0} jumpTo={() => () => {}} />);
expect(output).toEqual(null);
expect(pageList.html()).toEqual(null);
});
});

View File

@ -1,4 +1,5 @@
import React from 'react';
import PropTypes from 'prop-types';
import UnixTime from './UnixTime';
import ShortList from './ShortList';
import styles from './bootstrap.min.css';
@ -6,7 +7,7 @@ import cx from './cx';
class BusyWorkers extends React.Component {
static propTypes = {
worker: React.PropTypes.arrayOf(React.PropTypes.object).isRequired,
worker: PropTypes.arrayOf(PropTypes.object).isRequired,
}
render() {
@ -31,7 +32,7 @@ class BusyWorkers extends React.Component {
<td><UnixTime ts={worker.checkin_at}/></td>
<td>{worker.checkin}</td>
</tr>
);
);
})
}
</tbody>
@ -43,8 +44,8 @@ class BusyWorkers extends React.Component {
export default class Processes extends React.Component {
static propTypes = {
busyWorkerURL: React.PropTypes.string,
workerPoolURL: React.PropTypes.string,
busyWorkerURL: PropTypes.string,
workerPoolURL: PropTypes.string,
}
state = {
@ -135,7 +136,7 @@ export default class Processes extends React.Component {
</table>
</div>
</div>
);
);
})
}
</section>

View File

@ -1,17 +1,15 @@
import './TestSetup';
import expect from 'expect';
import Processes from './Processes';
import React from 'react';
import ReactTestUtils from 'react-addons-test-utils';
import { findAllByTag } from './TestUtils';
import { mount } from 'enzyme';
describe('Processes', () => {
it('shows workers', () => {
let r = ReactTestUtils.createRenderer();
r.render(<Processes />);
let processes = r.getMountedInstance();
let processes = mount(<Processes />);
expect(processes.state.busyWorker.length).toEqual(0);
expect(processes.state.workerPool.length).toEqual(0);
expect(processes.state().busyWorker.length).toEqual(0);
expect(processes.state().workerPool.length).toEqual(0);
processes.setState({
busyWorker: [
@ -40,16 +38,15 @@ describe('Processes', () => {
]
});
expect(processes.state.busyWorker.length).toEqual(1);
expect(processes.state.workerPool.length).toEqual(1);
expect(processes.workerCount).toEqual(3);
expect(processes.state().busyWorker.length).toEqual(1);
expect(processes.state().workerPool.length).toEqual(1);
expect(processes.instance().workerCount).toEqual(3);
const expectedBusyWorker = [ { args_json: '{}', checkin: '123', checkin_at: 1467753603, job_name: 'job1', started_at: 1467753603, worker_id: '2' } ];
let output = r.getRenderOutput();
let busyWorkers = findAllByTag(output, 'BusyWorkers');
let busyWorkers = processes.find('BusyWorkers');
expect(busyWorkers.length).toEqual(1);
expect(busyWorkers[0].props.worker).toEqual(expectedBusyWorker);
expect(processes.getBusyPoolWorker(processes.state.workerPool[0])).toEqual(expectedBusyWorker);
expect(busyWorkers.at(0).props().worker).toEqual(expectedBusyWorker);
expect(processes.instance().getBusyPoolWorker(processes.state().workerPool[0])).toEqual(expectedBusyWorker);
});
});

View File

@ -1,10 +1,11 @@
import React from 'react';
import PropTypes from 'prop-types';
import styles from './bootstrap.min.css';
import cx from './cx';
export default class Queues extends React.Component {
static propTypes = {
url: React.PropTypes.string,
url: PropTypes.string,
}
state = {
@ -53,7 +54,7 @@ export default class Queues extends React.Component {
<td>{queue.count}</td>
<td>{queue.latency}</td>
</tr>
);
);
})
}
</tbody>

View File

@ -1,14 +1,13 @@
import './TestSetup';
import expect from 'expect';
import Queues from './Queues';
import React from 'react';
import ReactTestUtils from 'react-addons-test-utils';
import { mount } from 'enzyme';
describe('Queues', () => {
it('gets queued count', () => {
let r = ReactTestUtils.createRenderer();
r.render(<Queues />);
let queues = r.getMountedInstance();
expect(queues.state.queues.length).toEqual(0);
let queues = mount(<Queues />);
expect(queues.state().queues.length).toEqual(0);
queues.setState({
queues: [
@ -17,7 +16,7 @@ describe('Queues', () => {
]
});
expect(queues.state.queues.length).toEqual(2);
expect(queues.queuedCount).toEqual(3);
expect(queues.state().queues.length).toEqual(2);
expect(queues.instance().queuedCount).toEqual(3);
});
});

View File

@ -1,4 +1,5 @@
import React from 'react';
import PropTypes from 'prop-types';
import PageList from './PageList';
import UnixTime from './UnixTime';
import styles from './bootstrap.min.css';
@ -6,7 +7,7 @@ import cx from './cx';
export default class RetryJobs extends React.Component {
static propTypes = {
url: React.PropTypes.string,
url: PropTypes.string,
}
state = {
@ -63,7 +64,7 @@ export default class RetryJobs extends React.Component {
<td>{job.err}</td>
<td><UnixTime ts={job.t} /></td>
</tr>
);
);
})
}
</tbody>

View File

@ -1,16 +1,14 @@
import './TestSetup';
import expect from 'expect';
import RetryJobs from './RetryJobs';
import React from 'react';
import ReactTestUtils from 'react-addons-test-utils';
import { findAllByTag } from './TestUtils';
import { mount } from 'enzyme';
describe('RetryJobs', () => {
it('shows jobs', () => {
let r = ReactTestUtils.createRenderer();
r.render(<RetryJobs />);
let retryJobs = r.getMountedInstance();
let retryJobs = mount(<RetryJobs />);
expect(retryJobs.state.jobs.length).toEqual(0);
expect(retryJobs.state().jobs.length).toEqual(0);
retryJobs.setState({
count: 2,
@ -20,13 +18,11 @@ describe('RetryJobs', () => {
]
});
expect(retryJobs.state.jobs.length).toEqual(2);
expect(retryJobs.state().jobs.length).toEqual(2);
});
it('has pages', () => {
let r = ReactTestUtils.createRenderer();
r.render(<RetryJobs />);
let retryJobs = r.getMountedInstance();
let retryJobs = mount(<RetryJobs />);
let genJob = (n) => {
let job = [];
@ -46,14 +42,13 @@ describe('RetryJobs', () => {
jobs: genJob(21)
});
expect(retryJobs.state.jobs.length).toEqual(21);
expect(retryJobs.state.page).toEqual(1);
expect(retryJobs.state().jobs.length).toEqual(21);
expect(retryJobs.state().page).toEqual(1);
let output = r.getRenderOutput();
let pageList = findAllByTag(output, 'PageList');
let pageList = retryJobs.find('PageList');
expect(pageList.length).toEqual(1);
pageList[0].props.jumpTo(2)();
expect(retryJobs.state.page).toEqual(2);
pageList.at(0).props().jumpTo(2)();
expect(retryJobs.state().page).toEqual(2);
});
});

View File

@ -1,4 +1,5 @@
import React from 'react';
import PropTypes from 'prop-types';
import PageList from './PageList';
import UnixTime from './UnixTime';
import styles from './bootstrap.min.css';
@ -6,7 +7,7 @@ import cx from './cx';
export default class ScheduledJobs extends React.Component {
static propTypes = {
url: React.PropTypes.string,
url: PropTypes.string,
}
state = {
@ -59,9 +60,9 @@ export default class ScheduledJobs extends React.Component {
<tr key={job.id}>
<td>{job.name}</td>
<td>{JSON.stringify(job.args)}</td>
<td><UnixTime ts={job.t} /></td>
<td><UnixTime ts={job.run_at} /></td>
</tr>
);
);
})
}
</tbody>

View File

@ -1,32 +1,28 @@
import './TestSetup';
import expect from 'expect';
import ScheduledJobs from './ScheduledJobs';
import React from 'react';
import ReactTestUtils from 'react-addons-test-utils';
import { findAllByTag } from './TestUtils';
import { mount } from 'enzyme';
describe('ScheduledJobs', () => {
it('shows jobs', () => {
let r = ReactTestUtils.createRenderer();
r.render(<ScheduledJobs />);
let scheduledJobs = r.getMountedInstance();
let scheduledJobs = mount(<ScheduledJobs />);
expect(scheduledJobs.state.jobs.length).toEqual(0);
expect(scheduledJobs.state().jobs.length).toEqual(0);
scheduledJobs.setState({
count: 2,
jobs: [
{id: 1, name: 'test', args: {}, t: 1467760821, err: 'err1'},
{id: 2, name: 'test2', args: {}, t: 1467760822, err: 'err2'}
{id: 1, name: 'test', args: {}, run_at: 1467760821, err: 'err1'},
{id: 2, name: 'test2', args: {}, run_at: 1467760822, err: 'err2'}
]
});
expect(scheduledJobs.state.jobs.length).toEqual(2);
expect(scheduledJobs.state().jobs.length).toEqual(2);
});
it('has pages', () => {
let r = ReactTestUtils.createRenderer();
r.render(<ScheduledJobs />);
let scheduledJobs = r.getMountedInstance();
let scheduledJobs = mount(<ScheduledJobs />);
let genJob = (n) => {
let job = [];
@ -35,7 +31,7 @@ describe('ScheduledJobs', () => {
id: i,
name: 'test',
args: {},
t: 1467760821,
run_at: 1467760821,
err: 'err',
});
}
@ -46,14 +42,13 @@ describe('ScheduledJobs', () => {
jobs: genJob(21)
});
expect(scheduledJobs.state.jobs.length).toEqual(21);
expect(scheduledJobs.state.page).toEqual(1);
expect(scheduledJobs.state().jobs.length).toEqual(21);
expect(scheduledJobs.state().page).toEqual(1);
let output = r.getRenderOutput();
let pageList = findAllByTag(output, 'PageList');
let pageList = scheduledJobs.find('PageList');
expect(pageList.length).toEqual(1);
pageList[0].props.jumpTo(2)();
expect(scheduledJobs.state.page).toEqual(2);
pageList.at(0).props().jumpTo(2)();
expect(scheduledJobs.state().page).toEqual(2);
});
});

View File

@ -1,9 +1,10 @@
import React from 'react';
import PropTypes from 'prop-types';
import styles from './ShortList.css';
export default class ShortList extends React.Component {
static propTypes = {
item: React.PropTypes.arrayOf(React.PropTypes.string).isRequired,
item: PropTypes.arrayOf(PropTypes.string).isRequired,
}
render() {

View File

@ -1,19 +1,18 @@
import './TestSetup';
import expect from 'expect';
import ShortList from './ShortList';
import React from 'react';
import ReactTestUtils from 'react-addons-test-utils';
import { mount } from 'enzyme';
describe('ShortList', () => {
it('lists items', () => {
let r = ReactTestUtils.createRenderer();
r.render(<ShortList item={['1', '2', '3', '4']} />);
let output = r.getRenderOutput();
let shortList = mount(<ShortList item={['1', '2', '3', '4']} />);
let ul = shortList.find('ul');
expect(output.type).toEqual('ul');
output.props.children.map((el, i) => {
ul.props().children.map((el, i) => {
expect(el.type).toEqual('li');
if (i < 3) {
expect(el.props.children).toEqual(i+1);
expect(el.props.children).toEqual(String(i+1));
} else {
expect(el.props.children).toEqual([i-2, ' more']);
}

View File

@ -0,0 +1,5 @@
import 'jsdom-global/register';
import { configure } from 'enzyme';
import Adapter from 'enzyme-adapter-react-16';
configure({ adapter: new Adapter() });

View File

@ -1,8 +1,9 @@
import React from 'react';
import PropTypes from 'prop-types';
export default class UnixTime extends React.Component {
static propTypes = {
ts: React.PropTypes.number.isRequired,
ts: PropTypes.number.isRequired,
}
render() {

View File

@ -1,16 +1,15 @@
import './TestSetup';
import expect from 'expect';
import UnixTime from './UnixTime';
import React from 'react';
import ReactTestUtils from 'react-addons-test-utils';
import { mount } from 'enzyme';
describe('UnixTime', () => {
it('formats human-readable time string', () => {
let r = ReactTestUtils.createRenderer();
r.render(<UnixTime ts={1467753603} />);
let output = r.getRenderOutput();
let output = mount(<UnixTime ts={1467753603} />);
expect(output.type).toEqual('time');
expect(output.props.children).toEqual('2016/07/05 21:20:03');
expect(output.props.dateTime).toEqual('2016-07-05T21:20:03.000Z');
let time = output.find('time');
expect(time.props().dateTime).toEqual('2016-07-05T21:20:03.000Z');
expect(time.text()).toEqual('2016/07/05 21:20:03');
});
});

View File

@ -1,4 +1,5 @@
import React from 'react';
import PropTypes from 'prop-types';
import { render } from 'react-dom';
import Processes from './Processes';
import DeadJobs from './DeadJobs';
@ -11,7 +12,7 @@ import cx from './cx';
class App extends React.Component {
static propTypes = {
children: React.PropTypes.element.isRequired,
children: PropTypes.element.isRequired,
}
render() {

View File

@ -11,12 +11,12 @@ module.exports = {
loaders: [
{
test: /\.js$/,
loader: 'babel',
loader: 'babel-loader',
exclude: /node_modules/,
},
{
test: /\.css$/,
loader: 'style!css?modules&camelCase&-url&localIdentName=[hash:base64:5]-[local]'
loader: 'style-loader!css-loader?modules&camelCase&-url&localIdentName=[hash:base64:5]-[local]'
},
{
test: /\.(png|woff|woff2|eot|ttf|svg)(\?[#a-z_]+)?$/,

View File

@ -24,12 +24,12 @@ module.exports = {
loaders: [
{
test: /\.js$/,
loader: 'babel',
loader: 'babel-loader',
exclude: /node_modules/,
},
{
test: /\.css$/,
loader: 'style!css?modules&camelCase&-url&localIdentName=[hash:base64:5]-[local]'
loader: 'style-loader!css-loader?modules&camelCase&-url&localIdentName=[hash:base64:5]-[local]'
},
{
test: /\.(png|woff|woff2|eot|ttf|svg)(\?[#a-z_]+)?$/,

File diff suppressed because it is too large Load Diff

View File

@ -8,10 +8,10 @@ import (
"sync"
"github.com/braintree/manners"
"github.com/garyburd/redigo/redis"
"github.com/gocraft/web"
"github.com/gocraft/work"
"github.com/gocraft/work/webui/internal/assets"
"github.com/gomodule/redigo/redis"
)
// Server implements an HTTP server which exposes a JSON API to view and manage gocraft/work items.

View File

@ -9,8 +9,8 @@ import (
"testing"
"time"
"github.com/garyburd/redigo/redis"
"github.com/gocraft/work"
"github.com/gomodule/redigo/redis"
"github.com/stretchr/testify/assert"
)

View File

@ -6,9 +6,11 @@ import (
"reflect"
"time"
"github.com/garyburd/redigo/redis"
"github.com/gomodule/redigo/redis"
)
const fetchKeysPerJobType = 6
type worker struct {
workerID string
poolID string
@ -59,11 +61,17 @@ func (w *worker) updateMiddlewareAndJobTypes(middleware []*middlewareHandler, jo
w.middleware = middleware
sampler := prioritySampler{}
for _, jt := range jobTypes {
sampler.add(jt.Priority, redisKeyJobs(w.namespace, jt.Name), redisKeyJobsInProgress(w.namespace, w.poolID, jt.Name))
sampler.add(jt.Priority,
redisKeyJobs(w.namespace, jt.Name),
redisKeyJobsInProgress(w.namespace, w.poolID, jt.Name),
redisKeyJobsPaused(w.namespace, jt.Name),
redisKeyJobsLock(w.namespace, jt.Name),
redisKeyJobsLockInfo(w.namespace, jt.Name),
redisKeyJobsConcurrency(w.namespace, jt.Name))
}
w.sampler = sampler
w.jobTypes = jobTypes
w.redisFetchScript = redis.NewScript(len(jobTypes)*2, redisLuaRpoplpushMultiCmd)
w.redisFetchScript = redis.NewScript(len(jobTypes)*fetchKeysPerJobType, redisLuaFetchJob)
}
func (w *worker) start() {
@ -103,29 +111,25 @@ func (w *worker) loop() {
drained = true
timer.Reset(0)
case <-timer.C:
gotJob := true
for gotJob {
job, err := w.fetchJob()
if err != nil {
logError("worker.fetch", err)
gotJob = false
timer.Reset(10 * time.Millisecond)
} else if job != nil {
w.processJob(job)
consequtiveNoJobs = 0
} else {
gotJob = false
if drained {
w.doneDrainingChan <- struct{}{}
drained = false
}
consequtiveNoJobs++
idx := consequtiveNoJobs
if idx >= int64(len(sleepBackoffsInMilliseconds)) {
idx = int64(len(sleepBackoffsInMilliseconds)) - 1
}
timer.Reset(time.Duration(sleepBackoffsInMilliseconds[idx]) * time.Millisecond)
job, err := w.fetchJob()
if err != nil {
logError("worker.fetch", err)
timer.Reset(10 * time.Millisecond)
} else if job != nil {
w.processJob(job)
consequtiveNoJobs = 0
timer.Reset(0)
} else {
if drained {
w.doneDrainingChan <- struct{}{}
drained = false
}
consequtiveNoJobs++
idx := consequtiveNoJobs
if idx >= int64(len(sleepBackoffsInMilliseconds)) {
idx = int64(len(sleepBackoffsInMilliseconds)) - 1
}
timer.Reset(time.Duration(sleepBackoffsInMilliseconds[idx]) * time.Millisecond)
}
}
}
@ -135,14 +139,16 @@ func (w *worker) fetchJob() (*Job, error) {
// resort queues
// NOTE: we could optimize this to only resort every second, or something.
w.sampler.sample()
numKeys := len(w.sampler.samples) * fetchKeysPerJobType
var scriptArgs = make([]interface{}, 0, numKeys+1)
var scriptArgs = make([]interface{}, 0, len(w.sampler.samples)*2)
for _, s := range w.sampler.samples {
scriptArgs = append(scriptArgs, s.redisJobs, s.redisJobsInProg)
scriptArgs = append(scriptArgs, s.redisJobs, s.redisJobsInProg, s.redisJobsPaused, s.redisJobsLock, s.redisJobsLockInfo, s.redisJobsMaxConcurrency) // KEYS[1-6 * N]
}
scriptArgs = append(scriptArgs, w.poolID) // ARGV[1]
conn := w.pool.Get()
defer conn.Close()
values, err := redis.Values(w.redisFetchScript.Do(conn, scriptArgs...))
if err == redis.ErrNil {
return nil, nil
@ -181,24 +187,24 @@ func (w *worker) processJob(job *Job) {
if job.Unique {
w.deleteUniqueJob(job)
}
if jt, ok := w.jobTypes[job.Name]; ok {
var runErr error
jt := w.jobTypes[job.Name]
if jt == nil {
runErr = fmt.Errorf("stray job: no handler")
logError("process_job.stray", runErr)
} else {
w.observeStarted(job.Name, job.ID, job.Args)
job.observer = w.observer // for Checkin
_, runErr := runJob(job, w.contextType, w.middleware, jt)
_, runErr = runJob(job, w.contextType, w.middleware, jt)
w.observeDone(job.Name, job.ID, runErr)
if runErr != nil {
job.failed(runErr)
w.addToRetryOrDead(jt, job, runErr)
} else {
w.removeJobFromInProgress(job)
}
} else {
// NOTE: since we don't have a jobType, we don't know max retries
runErr := fmt.Errorf("stray job: no handler")
logError("process_job.stray", runErr)
job.failed(runErr)
w.addToDead(job, runErr)
}
fate := terminateOnly
if runErr != nil {
job.failed(runErr)
fate = w.jobFate(jt, job)
}
w.removeJobFromInProgress(job, fate)
}
func (w *worker) deleteUniqueJob(job *Job) {
@ -215,73 +221,64 @@ func (w *worker) deleteUniqueJob(job *Job) {
}
}
func (w *worker) removeJobFromInProgress(job *Job) {
func (w *worker) removeJobFromInProgress(job *Job, fate terminateOp) {
conn := w.pool.Get()
defer conn.Close()
_, err := conn.Do("LREM", job.inProgQueue, 1, job.rawJSON)
if err != nil {
conn.Send("MULTI")
conn.Send("LREM", job.inProgQueue, 1, job.rawJSON)
conn.Send("DECR", redisKeyJobsLock(w.namespace, job.Name))
conn.Send("HINCRBY", redisKeyJobsLockInfo(w.namespace, job.Name), w.poolID, -1)
fate(conn)
if _, err := conn.Do("EXEC"); err != nil {
logError("worker.remove_job_from_in_progress.lrem", err)
}
}
func (w *worker) addToRetryOrDead(jt *jobType, job *Job, runErr error) {
failsRemaining := int64(jt.MaxFails) - job.Fails
if failsRemaining > 0 {
w.addToRetry(job, runErr)
} else {
if !jt.SkipDead {
w.addToDead(job, runErr)
type terminateOp func(conn redis.Conn)
func terminateOnly(_ redis.Conn) { return }
func terminateAndRetry(w *worker, jt *jobType, job *Job) terminateOp {
rawJSON, err := job.serialize()
if err != nil {
logError("worker.terminate_and_retry.serialize", err)
return terminateOnly
}
return func(conn redis.Conn) {
conn.Send("ZADD", redisKeyRetry(w.namespace), nowEpochSeconds()+jt.calcBackoff(job), rawJSON)
}
}
func terminateAndDead(w *worker, job *Job) terminateOp {
rawJSON, err := job.serialize()
if err != nil {
logError("worker.terminate_and_dead.serialize", err)
return terminateOnly
}
return func(conn redis.Conn) {
// NOTE: sidekiq limits the # of jobs: only keep jobs for 6 months, and only keep a max # of jobs
// The max # of jobs seems really horrible. Seems like operations should be on top of it.
// conn.Send("ZREMRANGEBYSCORE", redisKeyDead(w.namespace), "-inf", now - keepInterval)
// conn.Send("ZREMRANGEBYRANK", redisKeyDead(w.namespace), 0, -maxJobs)
conn.Send("ZADD", redisKeyDead(w.namespace), nowEpochSeconds(), rawJSON)
}
}
func (w *worker) jobFate(jt *jobType, job *Job) terminateOp {
if jt != nil {
failsRemaining := int64(jt.MaxFails) - job.Fails
if failsRemaining > 0 {
return terminateAndRetry(w, jt, job)
}
if jt.SkipDead {
return terminateOnly
}
}
return terminateAndDead(w, job)
}
func (w *worker) addToRetry(job *Job, runErr error) {
rawJSON, err := job.serialize()
if err != nil {
logError("worker.add_to_retry", err)
return
}
conn := w.pool.Get()
defer conn.Close()
conn.Send("MULTI")
conn.Send("LREM", job.inProgQueue, 1, job.rawJSON)
conn.Send("ZADD", redisKeyRetry(w.namespace), nowEpochSeconds()+backoff(job.Fails), rawJSON)
_, err = conn.Do("EXEC")
if err != nil {
logError("worker.add_to_retry.exec", err)
}
}
func (w *worker) addToDead(job *Job, runErr error) {
rawJSON, err := job.serialize()
if err != nil {
logError("worker.add_to_dead.serialize", err)
return
}
conn := w.pool.Get()
defer conn.Close()
// NOTE: sidekiq limits the # of jobs: only keep jobs for 6 months, and only keep a max # of jobs
// The max # of jobs seems really horrible. Seems like operations should be on top of it.
// conn.Send("ZREMRANGEBYSCORE", redisKeyDead(w.namespace), "-inf", now - keepInterval)
// conn.Send("ZREMRANGEBYRANK", redisKeyDead(w.namespace), 0, -maxJobs)
conn.Send("MULTI")
conn.Send("LREM", job.inProgQueue, 1, job.rawJSON)
conn.Send("ZADD", redisKeyDead(w.namespace), nowEpochSeconds(), rawJSON)
_, err = conn.Do("EXEC")
if err != nil {
logError("worker.add_to_dead.exec", err)
}
}
// backoff returns number of seconds t
func backoff(fails int64) int64 {
// Default algorithm returns an fastly increasing backoff counter which grows in an unbounded fashion
func defaultBackoffCalculator(job *Job) int64 {
fails := job.Fails
return (fails * fails * fails * fails) + 15 + (rand.Int63n(30) * (fails + 1))
}

View File

@ -1,12 +1,13 @@
package work
import (
"github.com/garyburd/redigo/redis"
"github.com/robfig/cron"
"reflect"
"sort"
"strings"
"sync"
"github.com/gomodule/redigo/redis"
"github.com/robfig/cron"
)
// WorkerPool represents a pool of workers. It forms the primary API of gocraft/work. WorkerPools provide the public API of gocraft/work. You can attach jobs and middlware to them. You can start and stop them. Based on their concurrency setting, they'll spin up N worker goroutines.
@ -39,11 +40,26 @@ type jobType struct {
DynamicHandler reflect.Value
}
func (jt *jobType) calcBackoff(j *Job) int64 {
if jt.Backoff == nil {
return defaultBackoffCalculator(j)
}
return jt.Backoff(j)
}
// You may provide your own backoff function for retrying failed jobs or use the builtin one.
// Returns the number of seconds to wait until the next attempt.
//
// The builtin backoff calculator provides an exponentially increasing wait function.
type BackoffCalculator func(job *Job) int64
// JobOptions can be passed to JobWithOptions.
type JobOptions struct {
Priority uint // Priority from 1 to 10000
MaxFails uint // 1: send straight to dead (unless SkipDead)
SkipDead bool // If true, don't send failed jobs to the dead queue when retries are exhausted.
Priority uint // Priority from 1 to 10000
MaxFails uint // 1: send straight to dead (unless SkipDead)
SkipDead bool // If true, don't send failed jobs to the dead queue when retries are exhausted.
MaxConcurrency uint // Max number of jobs to keep in flight (default is 0, meaning no max)
Backoff BackoffCalculator // If not set, uses the default backoff algorithm
}
// GenericHandler is a job handler without any custom context.
@ -61,7 +77,8 @@ type middlewareHandler struct {
GenericMiddlewareHandler GenericMiddlewareHandler
}
// NewWorkerPool creates a new worker pool. ctx should be a struct literal whose type will be used for middleware and handlers. concurrency specifies how many workers to spin up - each worker can process jobs concurrently.
// NewWorkerPool creates a new worker pool. ctx should be a struct literal whose type will be used for middleware and handlers.
// concurrency specifies how many workers to spin up - each worker can process jobs concurrently.
func NewWorkerPool(ctx interface{}, concurrency uint, namespace string, pool *redis.Pool) *WorkerPool {
if pool == nil {
panic("NewWorkerPool needs a non-nil *redis.Pool")
@ -111,7 +128,7 @@ func (wp *WorkerPool) Middleware(fn interface{}) *WorkerPool {
return wp
}
// Job registers the job name to the specified handler fn. For instnace, when workers pull jobs from the name queue, they'll be processed by the specified handler function.
// Job registers the job name to the specified handler fn. For instance, when workers pull jobs from the name queue they'll be processed by the specified handler function.
// fn can take one of these forms:
// (*ContextType).func(*Job) error, (ContextType matches the type of ctx specified when creating a pool)
// func(*Job) error, for the generic handler format.
@ -119,7 +136,8 @@ func (wp *WorkerPool) Job(name string, fn interface{}) *WorkerPool {
return wp.JobWithOptions(name, JobOptions{}, fn)
}
// JobWithOptions adds a handler for 'name' jobs as per the Job function, but permits you specify additional options such as a job's priority, retry count, and whether to send dead jobs to the dead job queue or trash them.
// JobWithOptions adds a handler for 'name' jobs as per the Job function, but permits you specify additional options
// such as a job's priority, retry count, and whether to send dead jobs to the dead job queue or trash them.
func (wp *WorkerPool) JobWithOptions(name string, jobOpts JobOptions, fn interface{}) *WorkerPool {
jobOpts = applyDefaultsAndValidate(jobOpts)
@ -166,7 +184,10 @@ func (wp *WorkerPool) Start() {
}
wp.started = true
// TODO: we should cleanup stale keys on startup from previously registered jobs
wp.writeConcurrencyControlsToRedis()
go wp.writeKnownJobsToRedis()
for _, w := range wp.workers {
go w.start()
}
@ -221,7 +242,7 @@ func (wp *WorkerPool) startRequeuers() {
}
wp.retrier = newRequeuer(wp.namespace, wp.pool, redisKeyRetry(wp.namespace), jobNames)
wp.scheduler = newRequeuer(wp.namespace, wp.pool, redisKeyScheduled(wp.namespace), jobNames)
wp.deadPoolReaper = newDeadPoolReaper(wp.namespace, wp.pool)
wp.deadPoolReaper = newDeadPoolReaper(wp.namespace, wp.pool, jobNames)
wp.retrier.start()
wp.scheduler.start()
wp.deadPoolReaper.start()
@ -243,27 +264,29 @@ func (wp *WorkerPool) writeKnownJobsToRedis() {
conn := wp.pool.Get()
defer conn.Close()
key := redisKeyKnownJobs(wp.namespace)
jobNames := make([]interface{}, 0, len(wp.jobTypes)+1)
jobNames = append(jobNames, key)
for k := range wp.jobTypes {
jobNames = append(jobNames, k)
}
_, err := conn.Do("SADD", jobNames...)
if err != nil {
if _, err := conn.Do("SADD", jobNames...); err != nil {
logError("write_known_jobs", err)
}
}
func newJobTypeGeneric(name string, opts JobOptions, handler GenericHandler) *jobType {
return &jobType{
Name: name,
JobOptions: opts,
IsGeneric: true,
GenericHandler: handler,
func (wp *WorkerPool) writeConcurrencyControlsToRedis() {
if len(wp.jobTypes) == 0 {
return
}
conn := wp.pool.Get()
defer conn.Close()
for jobName, jobType := range wp.jobTypes {
if _, err := conn.Do("SET", redisKeyJobsConcurrency(wp.namespace, jobName), jobType.MaxConcurrency); err != nil {
logError("write_concurrency_controls_max_concurrency", err)
}
}
}

View File

@ -3,9 +3,12 @@ package work
import (
"bytes"
"fmt"
"github.com/stretchr/testify/assert"
"reflect"
"testing"
"time"
"github.com/gomodule/redigo/redis"
"github.com/stretchr/testify/assert"
)
type tstCtx struct {
@ -110,3 +113,114 @@ func TestWorkerPoolValidations(t *testing.T) {
wp.Job("wat", TestWorkerPoolValidations)
}()
}
func TestWorkersPoolRunSingleThreaded(t *testing.T) {
pool := newTestPool(":6379")
ns := "work"
job1 := "job1"
numJobs, concurrency, sleepTime := 5, 5, 2
wp := setupTestWorkerPool(pool, ns, job1, concurrency, JobOptions{Priority: 1, MaxConcurrency: 1})
wp.Start()
// enqueue some jobs
enqueuer := NewEnqueuer(ns, pool)
for i := 0; i < numJobs; i++ {
_, err := enqueuer.Enqueue(job1, Q{"sleep": sleepTime})
assert.Nil(t, err)
}
// make sure we've enough jobs queued up to make an interesting test
jobsQueued := listSize(pool, redisKeyJobs(ns, job1))
assert.True(t, jobsQueued >= 3, "should be at least 3 jobs queued up, but only found %v", jobsQueued)
// now make sure the during the duration of job execution there is never > 1 job in flight
start := time.Now()
totalRuntime := time.Duration(sleepTime*numJobs) * time.Millisecond
time.Sleep(10 * time.Millisecond)
for time.Since(start) < totalRuntime {
// jobs in progress, lock count for the job and lock info for the pool should never exceed 1
jobsInProgress := listSize(pool, redisKeyJobsInProgress(ns, wp.workerPoolID, job1))
assert.True(t, jobsInProgress <= 1, "jobsInProgress should never exceed 1: actual=%d", jobsInProgress)
jobLockCount := getInt64(pool, redisKeyJobsLock(ns, job1))
assert.True(t, jobLockCount <= 1, "global lock count for job should never exceed 1, got: %v", jobLockCount)
wpLockCount := hgetInt64(pool, redisKeyJobsLockInfo(ns, job1), wp.workerPoolID)
assert.True(t, wpLockCount <= 1, "lock count for the worker pool should never exceed 1: actual=%v", wpLockCount)
time.Sleep(time.Duration(sleepTime) * time.Millisecond)
}
wp.Drain()
wp.Stop()
// At this point it should all be empty.
assert.EqualValues(t, 0, listSize(pool, redisKeyJobs(ns, job1)))
assert.EqualValues(t, 0, listSize(pool, redisKeyJobsInProgress(ns, wp.workerPoolID, job1)))
assert.EqualValues(t, 0, getInt64(pool, redisKeyJobsLock(ns, job1)))
assert.EqualValues(t, 0, hgetInt64(pool, redisKeyJobsLockInfo(ns, job1), wp.workerPoolID))
}
func TestWorkerPoolPauseSingleThreadedJobs(t *testing.T) {
pool := newTestPool(":6379")
ns, job1 := "work", "job1"
numJobs, concurrency, sleepTime := 5, 5, 2
wp := setupTestWorkerPool(pool, ns, job1, concurrency, JobOptions{Priority: 1, MaxConcurrency: 1})
wp.Start()
// enqueue some jobs
enqueuer := NewEnqueuer(ns, pool)
for i := 0; i < numJobs; i++ {
_, err := enqueuer.Enqueue(job1, Q{"sleep": sleepTime})
assert.Nil(t, err)
}
// provide time for jobs to process
time.Sleep(10 * time.Millisecond)
// pause work, provide time for outstanding jobs to finish and queue up another job
pauseJobs(ns, job1, pool)
time.Sleep(2 * time.Millisecond)
_, err := enqueuer.Enqueue(job1, Q{"sleep": sleepTime})
assert.Nil(t, err)
// check that we still have some jobs to process
assert.True(t, listSize(pool, redisKeyJobs(ns, job1)) >= 1)
// now make sure no jobs get started until we unpause
start := time.Now()
totalRuntime := time.Duration(sleepTime*numJobs) * time.Millisecond
for time.Since(start) < totalRuntime {
assert.EqualValues(t, 0, listSize(pool, redisKeyJobsInProgress(ns, wp.workerPoolID, job1)))
// lock count for the job and lock info for the pool should both be at 1 while job is running
assert.EqualValues(t, 0, getInt64(pool, redisKeyJobsLock(ns, job1)))
assert.EqualValues(t, 0, hgetInt64(pool, redisKeyJobsLockInfo(ns, job1), wp.workerPoolID))
time.Sleep(time.Duration(sleepTime) * time.Millisecond)
}
// unpause work and get past the backoff time
unpauseJobs(ns, job1, pool)
time.Sleep(10 * time.Millisecond)
wp.Drain()
wp.Stop()
// At this point it should all be empty.
assert.EqualValues(t, 0, listSize(pool, redisKeyJobs(ns, job1)))
assert.EqualValues(t, 0, listSize(pool, redisKeyJobsInProgress(ns, wp.workerPoolID, job1)))
assert.EqualValues(t, 0, getInt64(pool, redisKeyJobsLock(ns, job1)))
assert.EqualValues(t, 0, hgetInt64(pool, redisKeyJobsLockInfo(ns, job1), wp.workerPoolID))
}
// Test Helpers
func (t *TestContext) SleepyJob(job *Job) error {
sleepTime := time.Duration(job.ArgInt64("sleep"))
time.Sleep(sleepTime * time.Millisecond)
return nil
}
func setupTestWorkerPool(pool *redis.Pool, namespace, jobName string, concurrency int, jobOpts JobOptions) *WorkerPool {
deleteQueue(pool, namespace, jobName)
deleteRetryAndDead(pool, namespace)
deletePausedAndLockedKeys(namespace, jobName, pool)
wp := NewWorkerPool(TestContext{}, uint(concurrency), namespace, pool)
wp.JobWithOptions(jobName, jobOpts, (*TestContext).SleepyJob)
// reset the backoff times to help with testing
sleepBackoffsInMilliseconds = []int64{10, 10, 10, 10, 10}
return wp
}

View File

@ -3,10 +3,11 @@ package work
import (
"fmt"
"strconv"
"sync/atomic"
"testing"
"time"
"github.com/garyburd/redigo/redis"
"github.com/gomodule/redigo/redis"
"github.com/stretchr/testify/assert"
)
@ -93,6 +94,7 @@ func TestWorkerInProgress(t *testing.T) {
job1 := "job1"
deleteQueue(pool, ns, job1)
deleteRetryAndDead(pool, ns)
deletePausedAndLockedKeys(ns, job1, pool)
jobTypes := make(map[string]*jobType)
jobTypes[job1] = &jobType{
@ -117,6 +119,8 @@ func TestWorkerInProgress(t *testing.T) {
time.Sleep(10 * time.Millisecond)
assert.EqualValues(t, 0, listSize(pool, redisKeyJobs(ns, job1)))
assert.EqualValues(t, 1, listSize(pool, redisKeyJobsInProgress(ns, "1", job1)))
assert.EqualValues(t, 1, getInt64(pool, redisKeyJobsLock(ns, job1)))
assert.EqualValues(t, 1, hgetInt64(pool, redisKeyJobsLockInfo(ns, job1), w.poolID))
// nothing in the worker status
w.observer.drain()
@ -143,6 +147,7 @@ func TestWorkerRetry(t *testing.T) {
job1 := "job1"
deleteQueue(pool, ns, job1)
deleteRetryAndDead(pool, ns)
deletePausedAndLockedKeys(ns, job1, pool)
jobTypes := make(map[string]*jobType)
jobTypes[job1] = &jobType{
@ -167,6 +172,8 @@ func TestWorkerRetry(t *testing.T) {
assert.EqualValues(t, 0, zsetSize(pool, redisKeyDead(ns)))
assert.EqualValues(t, 0, listSize(pool, redisKeyJobs(ns, job1)))
assert.EqualValues(t, 0, listSize(pool, redisKeyJobsInProgress(ns, "1", job1)))
assert.EqualValues(t, 0, getInt64(pool, redisKeyJobsLock(ns, job1)))
assert.EqualValues(t, 0, hgetInt64(pool, redisKeyJobsLockInfo(ns, job1), w.poolID))
// Get the job on the retry queue
ts, job := jobOnZset(pool, redisKeyRetry(ns))
@ -180,6 +187,57 @@ func TestWorkerRetry(t *testing.T) {
assert.True(t, (nowEpochSeconds()-job.FailedAt) <= 2)
}
// Check if a custom backoff function functions functionally.
func TestWorkerRetryWithCustomBackoff(t *testing.T) {
pool := newTestPool(":6379")
ns := "work"
job1 := "job1"
deleteQueue(pool, ns, job1)
deleteRetryAndDead(pool, ns)
calledCustom := 0
custombo := func(job *Job) int64 {
calledCustom++
return 5 // Always 5 seconds
}
jobTypes := make(map[string]*jobType)
jobTypes[job1] = &jobType{
Name: job1,
JobOptions: JobOptions{Priority: 1, MaxFails: 3, Backoff: custombo},
IsGeneric: true,
GenericHandler: func(job *Job) error {
return fmt.Errorf("sorry kid")
},
}
enqueuer := NewEnqueuer(ns, pool)
_, err := enqueuer.Enqueue(job1, Q{"a": 1})
assert.Nil(t, err)
w := newWorker(ns, "1", pool, tstCtxType, nil, jobTypes)
w.start()
w.drain()
w.stop()
// Ensure the right stuff is in our queues:
assert.EqualValues(t, 1, zsetSize(pool, redisKeyRetry(ns)))
assert.EqualValues(t, 0, zsetSize(pool, redisKeyDead(ns)))
assert.EqualValues(t, 0, listSize(pool, redisKeyJobs(ns, job1)))
assert.EqualValues(t, 0, listSize(pool, redisKeyJobsInProgress(ns, "1", job1)))
// Get the job on the retry queue
ts, job := jobOnZset(pool, redisKeyRetry(ns))
assert.True(t, ts > nowEpochSeconds()) // enqueued in the future
assert.True(t, ts < (nowEpochSeconds()+10)) // but less than ten secs in
assert.Equal(t, job1, job.Name) // basics are preserved
assert.EqualValues(t, 1, job.Fails)
assert.Equal(t, "sorry kid", job.LastErr)
assert.True(t, (nowEpochSeconds()-job.FailedAt) <= 2)
assert.Equal(t, 1, calledCustom)
}
func TestWorkerDead(t *testing.T) {
pool := newTestPool(":6379")
ns := "work"
@ -188,6 +246,7 @@ func TestWorkerDead(t *testing.T) {
deleteQueue(pool, ns, job1)
deleteQueue(pool, ns, job2)
deleteRetryAndDead(pool, ns)
deletePausedAndLockedKeys(ns, job1, pool)
jobTypes := make(map[string]*jobType)
jobTypes[job1] = &jobType{
@ -220,10 +279,18 @@ func TestWorkerDead(t *testing.T) {
// Ensure the right stuff is in our queues:
assert.EqualValues(t, 0, zsetSize(pool, redisKeyRetry(ns)))
assert.EqualValues(t, 1, zsetSize(pool, redisKeyDead(ns)))
assert.EqualValues(t, 0, listSize(pool, redisKeyJobs(ns, job1)))
assert.EqualValues(t, 0, listSize(pool, redisKeyJobsInProgress(ns, "1", job1)))
assert.EqualValues(t, 0, getInt64(pool, redisKeyJobsLock(ns, job1)))
assert.EqualValues(t, 0, hgetInt64(pool, redisKeyJobsLockInfo(ns, job1), w.poolID))
// Get the job on the retry queue
assert.EqualValues(t, 0, listSize(pool, redisKeyJobs(ns, job2)))
assert.EqualValues(t, 0, listSize(pool, redisKeyJobsInProgress(ns, "1", job2)))
assert.EqualValues(t, 0, getInt64(pool, redisKeyJobsLock(ns, job2)))
assert.EqualValues(t, 0, hgetInt64(pool, redisKeyJobsLockInfo(ns, job2), w.poolID))
// Get the job on the dead queue
ts, job := jobOnZset(pool, redisKeyDead(ns))
assert.True(t, ts <= nowEpochSeconds())
@ -234,12 +301,74 @@ func TestWorkerDead(t *testing.T) {
assert.True(t, (nowEpochSeconds()-job.FailedAt) <= 2)
}
func TestWorkersPaused(t *testing.T) {
pool := newTestPool(":6379")
ns := "work"
job1 := "job1"
deleteQueue(pool, ns, job1)
deleteRetryAndDead(pool, ns)
deletePausedAndLockedKeys(ns, job1, pool)
jobTypes := make(map[string]*jobType)
jobTypes[job1] = &jobType{
Name: job1,
JobOptions: JobOptions{Priority: 1},
IsGeneric: true,
GenericHandler: func(job *Job) error {
time.Sleep(30 * time.Millisecond)
return nil
},
}
enqueuer := NewEnqueuer(ns, pool)
_, err := enqueuer.Enqueue(job1, Q{"a": 1})
assert.Nil(t, err)
w := newWorker(ns, "1", pool, tstCtxType, nil, jobTypes)
// pause the jobs prior to starting
err = pauseJobs(ns, job1, pool)
assert.Nil(t, err)
// reset the backoff times to help with testing
sleepBackoffsInMilliseconds = []int64{10, 10, 10, 10, 10}
w.start()
// make sure the jobs stay in the still in the run queue and not moved to in progress
for i := 0; i < 2; i++ {
time.Sleep(10 * time.Millisecond)
assert.EqualValues(t, 1, listSize(pool, redisKeyJobs(ns, job1)))
assert.EqualValues(t, 0, listSize(pool, redisKeyJobsInProgress(ns, "1", job1)))
}
// now unpause the jobs and check that they start
err = unpauseJobs(ns, job1, pool)
assert.Nil(t, err)
// sleep through 2 backoffs to make sure we allow enough time to start running
time.Sleep(20 * time.Millisecond)
assert.EqualValues(t, 0, listSize(pool, redisKeyJobs(ns, job1)))
assert.EqualValues(t, 1, listSize(pool, redisKeyJobsInProgress(ns, "1", job1)))
w.observer.drain()
h := readHash(pool, redisKeyWorkerObservation(ns, w.workerID))
assert.Equal(t, job1, h["job_name"])
assert.Equal(t, `{"a":1}`, h["args"])
w.drain()
w.stop()
// At this point, it should all be empty.
assert.EqualValues(t, 0, listSize(pool, redisKeyJobs(ns, job1)))
assert.EqualValues(t, 0, listSize(pool, redisKeyJobsInProgress(ns, "1", job1)))
// nothing in the worker status
h = readHash(pool, redisKeyWorkerObservation(ns, w.workerID))
assert.EqualValues(t, 0, len(h))
}
// Test that in the case of an unavailable Redis server,
// the worker loop exits in the case of a WorkerPool.Stop
func TestStop(t *testing.T) {
redisPool := &redis.Pool{
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("tcp", "notworking:6379")
c, err := redis.Dial("tcp", "notworking:6379", redis.DialConnectTimeout(1*time.Second))
if err != nil {
return nil, err
}
@ -318,7 +447,7 @@ func zsetSize(pool *redis.Pool, key string) int64 {
v, err := redis.Int64(conn.Do("ZCARD", key))
if err != nil {
panic("could not delete retry/dead queue: " + err.Error())
panic("could not get ZSET size: " + err.Error())
}
return v
}
@ -329,7 +458,29 @@ func listSize(pool *redis.Pool, key string) int64 {
v, err := redis.Int64(conn.Do("LLEN", key))
if err != nil {
panic("could not delete retry/dead queue: " + err.Error())
panic("could not get list length: " + err.Error())
}
return v
}
func getInt64(pool *redis.Pool, key string) int64 {
conn := pool.Get()
defer conn.Close()
v, err := redis.Int64(conn.Do("GET", key))
if err != nil {
panic("could not GET int64: " + err.Error())
}
return v
}
func hgetInt64(pool *redis.Pool, redisKey, hashKey string) int64 {
conn := pool.Get()
defer conn.Close()
v, err := redis.Int64(conn.Do("HGET", redisKey, hashKey))
if err != nil {
panic("could not HGET int64: " + err.Error())
}
return v
}
@ -340,7 +491,7 @@ func jobOnZset(pool *redis.Pool, key string) (int64, *Job) {
v, err := conn.Do("ZRANGE", key, 0, 0, "WITHSCORES")
if err != nil {
panic("could not delete retry/dead queue: " + err.Error())
panic("ZRANGE error: " + err.Error())
}
vv := v.([]interface{})
@ -365,7 +516,7 @@ func jobOnQueue(pool *redis.Pool, key string) *Job {
rawJSON, err := redis.Bytes(conn.Do("RPOP", key))
if err != nil {
panic("could not delete retry/dead queue: " + err.Error())
panic("could RPOP from job queue: " + err.Error())
}
job, err := newJob(rawJSON, nil, nil)
@ -401,3 +552,82 @@ func cleanKeyspace(namespace string, pool *redis.Pool) {
}
}
}
func pauseJobs(namespace, jobName string, pool *redis.Pool) error {
conn := pool.Get()
defer conn.Close()
if _, err := conn.Do("SET", redisKeyJobsPaused(namespace, jobName), "1"); err != nil {
return err
}
return nil
}
func unpauseJobs(namespace, jobName string, pool *redis.Pool) error {
conn := pool.Get()
defer conn.Close()
if _, err := conn.Do("DEL", redisKeyJobsPaused(namespace, jobName)); err != nil {
return err
}
return nil
}
func deletePausedAndLockedKeys(namespace, jobName string, pool *redis.Pool) error {
conn := pool.Get()
defer conn.Close()
if _, err := conn.Do("DEL", redisKeyJobsPaused(namespace, jobName)); err != nil {
return err
}
if _, err := conn.Do("DEL", redisKeyJobsLock(namespace, jobName)); err != nil {
return err
}
if _, err := conn.Do("DEL", redisKeyJobsLockInfo(namespace, jobName)); err != nil {
return err
}
return nil
}
type emptyCtx struct{}
// Starts up a pool with two workers emptying it as fast as they can
// The pool is Stop()ped while jobs are still going on. Tests that the
// pool processing is really stopped and that it's not first completely
// drained before returning.
// https://github.com/gocraft/work/issues/24
func TestWorkerPoolStop(t *testing.T) {
ns := "will_it_end"
pool := newTestPool(":6379")
var started, stopped int32
num_iters := 30
wp := NewWorkerPool(emptyCtx{}, 2, ns, pool)
wp.Job("sample_job", func(c *emptyCtx, job *Job) error {
atomic.AddInt32(&started, 1)
time.Sleep(1 * time.Second)
atomic.AddInt32(&stopped, 1)
return nil
})
var enqueuer = NewEnqueuer(ns, pool)
for i := 0; i <= num_iters; i++ {
enqueuer.Enqueue("sample_job", Q{})
}
// Start the pool and quit before it has had a chance to complete
// all the jobs.
wp.Start()
time.Sleep(5 * time.Second)
wp.Stop()
if started != stopped {
t.Errorf("Expected that jobs were finished and not killed while processing (started=%d, stopped=%d)", started, stopped)
}
if started >= int32(num_iters) {
t.Errorf("Expected that jobs queue was not completely emptied.")
}
}

View File

@ -0,0 +1,5 @@
Ask questions at
[StackOverflow](https://stackoverflow.com/questions/ask?tags=go+redis).
[Open an issue](https://github.com/garyburd/redigo/issues/new) to discuss your
plans before doing any work on Redigo.

View File

@ -0,0 +1 @@
Ask questions at https://stackoverflow.com/questions/ask?tags=go+redis

20
src/vendor/github.com/gomodule/redigo/.travis.yml generated vendored Normal file
View File

@ -0,0 +1,20 @@
language: go
sudo: false
services:
- redis-server
go:
- 1.4
- 1.5.x
- 1.6.x
- 1.7.x
- 1.8.x
- 1.9.x
- 1.10.x
- tip
script:
- go get -t -v ./...
- diff -u <(echo -n) <(gofmt -d .)
- go vet $(go list ./... | grep -v /vendor/)
- go test -v -race ./...

175
src/vendor/github.com/gomodule/redigo/LICENSE generated vendored Normal file
View File

@ -0,0 +1,175 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.

51
src/vendor/github.com/gomodule/redigo/README.markdown generated vendored Normal file
View File

@ -0,0 +1,51 @@
Redigo
======
[![Build Status](https://travis-ci.org/gomodule/redigo.svg?branch=master)](https://travis-ci.org/gomodule/redigo)
[![GoDoc](https://godoc.org/github.com/gomodule/redigo/redis?status.svg)](https://godoc.org/github.com/gomodule/redigo/redis)
Redigo is a [Go](http://golang.org/) client for the [Redis](http://redis.io/) database.
Features
-------
* A [Print-like](http://godoc.org/github.com/gomodule/redigo/redis#hdr-Executing_Commands) API with support for all Redis commands.
* [Pipelining](http://godoc.org/github.com/gomodule/redigo/redis#hdr-Pipelining), including pipelined transactions.
* [Publish/Subscribe](http://godoc.org/github.com/gomodule/redigo/redis#hdr-Publish_and_Subscribe).
* [Connection pooling](http://godoc.org/github.com/gomodule/redigo/redis#Pool).
* [Script helper type](http://godoc.org/github.com/gomodule/redigo/redis#Script) with optimistic use of EVALSHA.
* [Helper functions](http://godoc.org/github.com/gomodule/redigo/redis#hdr-Reply_Helpers) for working with command replies.
Documentation
-------------
- [API Reference](http://godoc.org/github.com/gomodule/redigo/redis)
- [FAQ](https://github.com/gomodule/redigo/wiki/FAQ)
- [Examples](https://godoc.org/github.com/gomodule/redigo/redis#pkg-examples)
Installation
------------
Install Redigo using the "go get" command:
go get github.com/gomodule/redigo/redis
The Go distribution is Redigo's only dependency.
Related Projects
----------------
- [rafaeljusto/redigomock](https://godoc.org/github.com/rafaeljusto/redigomock) - A mock library for Redigo.
- [chasex/redis-go-cluster](https://github.com/chasex/redis-go-cluster) - A Redis cluster client implementation.
- [FZambia/go-sentinel](https://github.com/FZambia/go-sentinel) - Redis Sentinel support for Redigo
- [PuerkitoBio/redisc](https://github.com/PuerkitoBio/redisc) - Redis Cluster client built on top of Redigo
Contributing
------------
See [CONTRIBUTING.md](https://github.com/gomodule/redigo/blob/master/.github/CONTRIBUTING.md).
License
-------
Redigo is available under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0.html).

View File

@ -0,0 +1,54 @@
// Copyright 2014 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package internal // import "github.com/gomodule/redigo/internal"
import (
"strings"
)
const (
WatchState = 1 << iota
MultiState
SubscribeState
MonitorState
)
type CommandInfo struct {
Set, Clear int
}
var commandInfos = map[string]CommandInfo{
"WATCH": {Set: WatchState},
"UNWATCH": {Clear: WatchState},
"MULTI": {Set: MultiState},
"EXEC": {Clear: WatchState | MultiState},
"DISCARD": {Clear: WatchState | MultiState},
"PSUBSCRIBE": {Set: SubscribeState},
"SUBSCRIBE": {Set: SubscribeState},
"MONITOR": {Set: MonitorState},
}
func init() {
for n, ci := range commandInfos {
commandInfos[strings.ToLower(n)] = ci
}
}
func LookupCommandInfo(commandName string) CommandInfo {
if ci, ok := commandInfos[commandName]; ok {
return ci
}
return commandInfos[strings.ToUpper(commandName)]
}

View File

@ -0,0 +1,27 @@
package internal
import "testing"
func TestLookupCommandInfo(t *testing.T) {
for _, n := range []string{"watch", "WATCH", "wAtch"} {
if LookupCommandInfo(n) == (CommandInfo{}) {
t.Errorf("LookupCommandInfo(%q) = CommandInfo{}, expected non-zero value", n)
}
}
}
func benchmarkLookupCommandInfo(b *testing.B, names ...string) {
for i := 0; i < b.N; i++ {
for _, c := range names {
LookupCommandInfo(c)
}
}
}
func BenchmarkLookupCommandInfoCorrectCase(b *testing.B) {
benchmarkLookupCommandInfo(b, "watch", "WATCH", "monitor", "MONITOR")
}
func BenchmarkLookupCommandInfoMixedCase(b *testing.B) {
benchmarkLookupCommandInfo(b, "wAtch", "WeTCH", "monItor", "MONiTOR")
}

View File

@ -0,0 +1,68 @@
// Copyright 2014 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
// Package redistest contains utilities for writing Redigo tests.
package redistest
import (
"errors"
"time"
"github.com/gomodule/redigo/redis"
)
type testConn struct {
redis.Conn
}
func (t testConn) Close() error {
_, err := t.Conn.Do("SELECT", "9")
if err != nil {
return nil
}
_, err = t.Conn.Do("FLUSHDB")
if err != nil {
return err
}
return t.Conn.Close()
}
// Dial dials the local Redis server and selects database 9. To prevent
// stomping on real data, DialTestDB fails if database 9 contains data. The
// returned connection flushes database 9 on close.
func Dial() (redis.Conn, error) {
c, err := redis.DialTimeout("tcp", ":6379", 0, 1*time.Second, 1*time.Second)
if err != nil {
return nil, err
}
_, err = c.Do("SELECT", "9")
if err != nil {
c.Close()
return nil, err
}
n, err := redis.Int(c.Do("DBSIZE"))
if err != nil {
c.Close()
return nil, err
}
if n != 0 {
c.Close()
return nil, errors.New("database #9 is not empty, test can not continue")
}
return testConn{c}, nil
}

673
src/vendor/github.com/gomodule/redigo/redis/conn.go generated vendored Normal file
View File

@ -0,0 +1,673 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis
import (
"bufio"
"bytes"
"crypto/tls"
"errors"
"fmt"
"io"
"net"
"net/url"
"regexp"
"strconv"
"sync"
"time"
)
var (
_ ConnWithTimeout = (*conn)(nil)
)
// conn is the low-level implementation of Conn
type conn struct {
// Shared
mu sync.Mutex
pending int
err error
conn net.Conn
// Read
readTimeout time.Duration
br *bufio.Reader
// Write
writeTimeout time.Duration
bw *bufio.Writer
// Scratch space for formatting argument length.
// '*' or '$', length, "\r\n"
lenScratch [32]byte
// Scratch space for formatting integers and floats.
numScratch [40]byte
}
// DialTimeout acts like Dial but takes timeouts for establishing the
// connection to the server, writing a command and reading a reply.
//
// Deprecated: Use Dial with options instead.
func DialTimeout(network, address string, connectTimeout, readTimeout, writeTimeout time.Duration) (Conn, error) {
return Dial(network, address,
DialConnectTimeout(connectTimeout),
DialReadTimeout(readTimeout),
DialWriteTimeout(writeTimeout))
}
// DialOption specifies an option for dialing a Redis server.
type DialOption struct {
f func(*dialOptions)
}
type dialOptions struct {
readTimeout time.Duration
writeTimeout time.Duration
dialer *net.Dialer
dial func(network, addr string) (net.Conn, error)
db int
password string
useTLS bool
skipVerify bool
tlsConfig *tls.Config
}
// DialReadTimeout specifies the timeout for reading a single command reply.
func DialReadTimeout(d time.Duration) DialOption {
return DialOption{func(do *dialOptions) {
do.readTimeout = d
}}
}
// DialWriteTimeout specifies the timeout for writing a single command.
func DialWriteTimeout(d time.Duration) DialOption {
return DialOption{func(do *dialOptions) {
do.writeTimeout = d
}}
}
// DialConnectTimeout specifies the timeout for connecting to the Redis server when
// no DialNetDial option is specified.
func DialConnectTimeout(d time.Duration) DialOption {
return DialOption{func(do *dialOptions) {
do.dialer.Timeout = d
}}
}
// DialKeepAlive specifies the keep-alive period for TCP connections to the Redis server
// when no DialNetDial option is specified.
// If zero, keep-alives are not enabled. If no DialKeepAlive option is specified then
// the default of 5 minutes is used to ensure that half-closed TCP sessions are detected.
func DialKeepAlive(d time.Duration) DialOption {
return DialOption{func(do *dialOptions) {
do.dialer.KeepAlive = d
}}
}
// DialNetDial specifies a custom dial function for creating TCP
// connections, otherwise a net.Dialer customized via the other options is used.
// DialNetDial overrides DialConnectTimeout and DialKeepAlive.
func DialNetDial(dial func(network, addr string) (net.Conn, error)) DialOption {
return DialOption{func(do *dialOptions) {
do.dial = dial
}}
}
// DialDatabase specifies the database to select when dialing a connection.
func DialDatabase(db int) DialOption {
return DialOption{func(do *dialOptions) {
do.db = db
}}
}
// DialPassword specifies the password to use when connecting to
// the Redis server.
func DialPassword(password string) DialOption {
return DialOption{func(do *dialOptions) {
do.password = password
}}
}
// DialTLSConfig specifies the config to use when a TLS connection is dialed.
// Has no effect when not dialing a TLS connection.
func DialTLSConfig(c *tls.Config) DialOption {
return DialOption{func(do *dialOptions) {
do.tlsConfig = c
}}
}
// DialTLSSkipVerify disables server name verification when connecting over
// TLS. Has no effect when not dialing a TLS connection.
func DialTLSSkipVerify(skip bool) DialOption {
return DialOption{func(do *dialOptions) {
do.skipVerify = skip
}}
}
// DialUseTLS specifies whether TLS should be used when connecting to the
// server. This option is ignore by DialURL.
func DialUseTLS(useTLS bool) DialOption {
return DialOption{func(do *dialOptions) {
do.useTLS = useTLS
}}
}
// Dial connects to the Redis server at the given network and
// address using the specified options.
func Dial(network, address string, options ...DialOption) (Conn, error) {
do := dialOptions{
dialer: &net.Dialer{
KeepAlive: time.Minute * 5,
},
}
for _, option := range options {
option.f(&do)
}
if do.dial == nil {
do.dial = do.dialer.Dial
}
netConn, err := do.dial(network, address)
if err != nil {
return nil, err
}
if do.useTLS {
var tlsConfig *tls.Config
if do.tlsConfig == nil {
tlsConfig = &tls.Config{InsecureSkipVerify: do.skipVerify}
} else {
tlsConfig = cloneTLSConfig(do.tlsConfig)
}
if tlsConfig.ServerName == "" {
host, _, err := net.SplitHostPort(address)
if err != nil {
netConn.Close()
return nil, err
}
tlsConfig.ServerName = host
}
tlsConn := tls.Client(netConn, tlsConfig)
if err := tlsConn.Handshake(); err != nil {
netConn.Close()
return nil, err
}
netConn = tlsConn
}
c := &conn{
conn: netConn,
bw: bufio.NewWriter(netConn),
br: bufio.NewReader(netConn),
readTimeout: do.readTimeout,
writeTimeout: do.writeTimeout,
}
if do.password != "" {
if _, err := c.Do("AUTH", do.password); err != nil {
netConn.Close()
return nil, err
}
}
if do.db != 0 {
if _, err := c.Do("SELECT", do.db); err != nil {
netConn.Close()
return nil, err
}
}
return c, nil
}
var pathDBRegexp = regexp.MustCompile(`/(\d*)\z`)
// DialURL connects to a Redis server at the given URL using the Redis
// URI scheme. URLs should follow the draft IANA specification for the
// scheme (https://www.iana.org/assignments/uri-schemes/prov/redis).
func DialURL(rawurl string, options ...DialOption) (Conn, error) {
u, err := url.Parse(rawurl)
if err != nil {
return nil, err
}
if u.Scheme != "redis" && u.Scheme != "rediss" {
return nil, fmt.Errorf("invalid redis URL scheme: %s", u.Scheme)
}
// As per the IANA draft spec, the host defaults to localhost and
// the port defaults to 6379.
host, port, err := net.SplitHostPort(u.Host)
if err != nil {
// assume port is missing
host = u.Host
port = "6379"
}
if host == "" {
host = "localhost"
}
address := net.JoinHostPort(host, port)
if u.User != nil {
password, isSet := u.User.Password()
if isSet {
options = append(options, DialPassword(password))
}
}
match := pathDBRegexp.FindStringSubmatch(u.Path)
if len(match) == 2 {
db := 0
if len(match[1]) > 0 {
db, err = strconv.Atoi(match[1])
if err != nil {
return nil, fmt.Errorf("invalid database: %s", u.Path[1:])
}
}
if db != 0 {
options = append(options, DialDatabase(db))
}
} else if u.Path != "" {
return nil, fmt.Errorf("invalid database: %s", u.Path[1:])
}
options = append(options, DialUseTLS(u.Scheme == "rediss"))
return Dial("tcp", address, options...)
}
// NewConn returns a new Redigo connection for the given net connection.
func NewConn(netConn net.Conn, readTimeout, writeTimeout time.Duration) Conn {
return &conn{
conn: netConn,
bw: bufio.NewWriter(netConn),
br: bufio.NewReader(netConn),
readTimeout: readTimeout,
writeTimeout: writeTimeout,
}
}
func (c *conn) Close() error {
c.mu.Lock()
err := c.err
if c.err == nil {
c.err = errors.New("redigo: closed")
err = c.conn.Close()
}
c.mu.Unlock()
return err
}
func (c *conn) fatal(err error) error {
c.mu.Lock()
if c.err == nil {
c.err = err
// Close connection to force errors on subsequent calls and to unblock
// other reader or writer.
c.conn.Close()
}
c.mu.Unlock()
return err
}
func (c *conn) Err() error {
c.mu.Lock()
err := c.err
c.mu.Unlock()
return err
}
func (c *conn) writeLen(prefix byte, n int) error {
c.lenScratch[len(c.lenScratch)-1] = '\n'
c.lenScratch[len(c.lenScratch)-2] = '\r'
i := len(c.lenScratch) - 3
for {
c.lenScratch[i] = byte('0' + n%10)
i -= 1
n = n / 10
if n == 0 {
break
}
}
c.lenScratch[i] = prefix
_, err := c.bw.Write(c.lenScratch[i:])
return err
}
func (c *conn) writeString(s string) error {
c.writeLen('$', len(s))
c.bw.WriteString(s)
_, err := c.bw.WriteString("\r\n")
return err
}
func (c *conn) writeBytes(p []byte) error {
c.writeLen('$', len(p))
c.bw.Write(p)
_, err := c.bw.WriteString("\r\n")
return err
}
func (c *conn) writeInt64(n int64) error {
return c.writeBytes(strconv.AppendInt(c.numScratch[:0], n, 10))
}
func (c *conn) writeFloat64(n float64) error {
return c.writeBytes(strconv.AppendFloat(c.numScratch[:0], n, 'g', -1, 64))
}
func (c *conn) writeCommand(cmd string, args []interface{}) error {
c.writeLen('*', 1+len(args))
if err := c.writeString(cmd); err != nil {
return err
}
for _, arg := range args {
if err := c.writeArg(arg, true); err != nil {
return err
}
}
return nil
}
func (c *conn) writeArg(arg interface{}, argumentTypeOK bool) (err error) {
switch arg := arg.(type) {
case string:
return c.writeString(arg)
case []byte:
return c.writeBytes(arg)
case int:
return c.writeInt64(int64(arg))
case int64:
return c.writeInt64(arg)
case float64:
return c.writeFloat64(arg)
case bool:
if arg {
return c.writeString("1")
} else {
return c.writeString("0")
}
case nil:
return c.writeString("")
case Argument:
if argumentTypeOK {
return c.writeArg(arg.RedisArg(), false)
}
// See comment in default clause below.
var buf bytes.Buffer
fmt.Fprint(&buf, arg)
return c.writeBytes(buf.Bytes())
default:
// This default clause is intended to handle builtin numeric types.
// The function should return an error for other types, but this is not
// done for compatibility with previous versions of the package.
var buf bytes.Buffer
fmt.Fprint(&buf, arg)
return c.writeBytes(buf.Bytes())
}
}
type protocolError string
func (pe protocolError) Error() string {
return fmt.Sprintf("redigo: %s (possible server error or unsupported concurrent read by application)", string(pe))
}
func (c *conn) readLine() ([]byte, error) {
p, err := c.br.ReadSlice('\n')
if err == bufio.ErrBufferFull {
return nil, protocolError("long response line")
}
if err != nil {
return nil, err
}
i := len(p) - 2
if i < 0 || p[i] != '\r' {
return nil, protocolError("bad response line terminator")
}
return p[:i], nil
}
// parseLen parses bulk string and array lengths.
func parseLen(p []byte) (int, error) {
if len(p) == 0 {
return -1, protocolError("malformed length")
}
if p[0] == '-' && len(p) == 2 && p[1] == '1' {
// handle $-1 and $-1 null replies.
return -1, nil
}
var n int
for _, b := range p {
n *= 10
if b < '0' || b > '9' {
return -1, protocolError("illegal bytes in length")
}
n += int(b - '0')
}
return n, nil
}
// parseInt parses an integer reply.
func parseInt(p []byte) (interface{}, error) {
if len(p) == 0 {
return 0, protocolError("malformed integer")
}
var negate bool
if p[0] == '-' {
negate = true
p = p[1:]
if len(p) == 0 {
return 0, protocolError("malformed integer")
}
}
var n int64
for _, b := range p {
n *= 10
if b < '0' || b > '9' {
return 0, protocolError("illegal bytes in length")
}
n += int64(b - '0')
}
if negate {
n = -n
}
return n, nil
}
var (
okReply interface{} = "OK"
pongReply interface{} = "PONG"
)
func (c *conn) readReply() (interface{}, error) {
line, err := c.readLine()
if err != nil {
return nil, err
}
if len(line) == 0 {
return nil, protocolError("short response line")
}
switch line[0] {
case '+':
switch {
case len(line) == 3 && line[1] == 'O' && line[2] == 'K':
// Avoid allocation for frequent "+OK" response.
return okReply, nil
case len(line) == 5 && line[1] == 'P' && line[2] == 'O' && line[3] == 'N' && line[4] == 'G':
// Avoid allocation in PING command benchmarks :)
return pongReply, nil
default:
return string(line[1:]), nil
}
case '-':
return Error(string(line[1:])), nil
case ':':
return parseInt(line[1:])
case '$':
n, err := parseLen(line[1:])
if n < 0 || err != nil {
return nil, err
}
p := make([]byte, n)
_, err = io.ReadFull(c.br, p)
if err != nil {
return nil, err
}
if line, err := c.readLine(); err != nil {
return nil, err
} else if len(line) != 0 {
return nil, protocolError("bad bulk string format")
}
return p, nil
case '*':
n, err := parseLen(line[1:])
if n < 0 || err != nil {
return nil, err
}
r := make([]interface{}, n)
for i := range r {
r[i], err = c.readReply()
if err != nil {
return nil, err
}
}
return r, nil
}
return nil, protocolError("unexpected response line")
}
func (c *conn) Send(cmd string, args ...interface{}) error {
c.mu.Lock()
c.pending += 1
c.mu.Unlock()
if c.writeTimeout != 0 {
c.conn.SetWriteDeadline(time.Now().Add(c.writeTimeout))
}
if err := c.writeCommand(cmd, args); err != nil {
return c.fatal(err)
}
return nil
}
func (c *conn) Flush() error {
if c.writeTimeout != 0 {
c.conn.SetWriteDeadline(time.Now().Add(c.writeTimeout))
}
if err := c.bw.Flush(); err != nil {
return c.fatal(err)
}
return nil
}
func (c *conn) Receive() (interface{}, error) {
return c.ReceiveWithTimeout(c.readTimeout)
}
func (c *conn) ReceiveWithTimeout(timeout time.Duration) (reply interface{}, err error) {
var deadline time.Time
if timeout != 0 {
deadline = time.Now().Add(timeout)
}
c.conn.SetReadDeadline(deadline)
if reply, err = c.readReply(); err != nil {
return nil, c.fatal(err)
}
// When using pub/sub, the number of receives can be greater than the
// number of sends. To enable normal use of the connection after
// unsubscribing from all channels, we do not decrement pending to a
// negative value.
//
// The pending field is decremented after the reply is read to handle the
// case where Receive is called before Send.
c.mu.Lock()
if c.pending > 0 {
c.pending -= 1
}
c.mu.Unlock()
if err, ok := reply.(Error); ok {
return nil, err
}
return
}
func (c *conn) Do(cmd string, args ...interface{}) (interface{}, error) {
return c.DoWithTimeout(c.readTimeout, cmd, args...)
}
func (c *conn) DoWithTimeout(readTimeout time.Duration, cmd string, args ...interface{}) (interface{}, error) {
c.mu.Lock()
pending := c.pending
c.pending = 0
c.mu.Unlock()
if cmd == "" && pending == 0 {
return nil, nil
}
if c.writeTimeout != 0 {
c.conn.SetWriteDeadline(time.Now().Add(c.writeTimeout))
}
if cmd != "" {
if err := c.writeCommand(cmd, args); err != nil {
return nil, c.fatal(err)
}
}
if err := c.bw.Flush(); err != nil {
return nil, c.fatal(err)
}
var deadline time.Time
if readTimeout != 0 {
deadline = time.Now().Add(readTimeout)
}
c.conn.SetReadDeadline(deadline)
if cmd == "" {
reply := make([]interface{}, pending)
for i := range reply {
r, e := c.readReply()
if e != nil {
return nil, c.fatal(e)
}
reply[i] = r
}
return reply, nil
}
var err error
var reply interface{}
for i := 0; i <= pending; i++ {
var e error
if reply, e = c.readReply(); e != nil {
return nil, c.fatal(e)
}
if e, ok := reply.(Error); ok && err == nil {
err = e
}
}
return reply, err
}

View File

@ -0,0 +1,867 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis_test
import (
"bytes"
"crypto/tls"
"crypto/x509"
"fmt"
"io"
"math"
"net"
"os"
"reflect"
"strings"
"testing"
"time"
"github.com/gomodule/redigo/redis"
)
type testConn struct {
io.Reader
io.Writer
readDeadline time.Time
writeDeadline time.Time
}
func (*testConn) Close() error { return nil }
func (*testConn) LocalAddr() net.Addr { return nil }
func (*testConn) RemoteAddr() net.Addr { return nil }
func (c *testConn) SetDeadline(t time.Time) error { c.readDeadline = t; c.writeDeadline = t; return nil }
func (c *testConn) SetReadDeadline(t time.Time) error { c.readDeadline = t; return nil }
func (c *testConn) SetWriteDeadline(t time.Time) error { c.writeDeadline = t; return nil }
func dialTestConn(r string, w io.Writer) redis.DialOption {
return redis.DialNetDial(func(network, addr string) (net.Conn, error) {
return &testConn{Reader: strings.NewReader(r), Writer: w}, nil
})
}
type tlsTestConn struct {
net.Conn
done chan struct{}
}
func (c *tlsTestConn) Close() error {
c.Conn.Close()
<-c.done
return nil
}
func dialTestConnTLS(r string, w io.Writer) redis.DialOption {
return redis.DialNetDial(func(network, addr string) (net.Conn, error) {
client, server := net.Pipe()
tlsServer := tls.Server(server, &serverTLSConfig)
go io.Copy(tlsServer, strings.NewReader(r))
done := make(chan struct{})
go func() {
io.Copy(w, tlsServer)
close(done)
}()
return &tlsTestConn{Conn: client, done: done}, nil
})
}
type durationArg struct {
time.Duration
}
func (t durationArg) RedisArg() interface{} {
return t.Seconds()
}
type recursiveArg int
func (v recursiveArg) RedisArg() interface{} { return v }
var writeTests = []struct {
args []interface{}
expected string
}{
{
[]interface{}{"SET", "key", "value"},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$5\r\nvalue\r\n",
},
{
[]interface{}{"SET", "key", "value"},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$5\r\nvalue\r\n",
},
{
[]interface{}{"SET", "key", byte(100)},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$3\r\n100\r\n",
},
{
[]interface{}{"SET", "key", 100},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$3\r\n100\r\n",
},
{
[]interface{}{"SET", "key", int64(math.MinInt64)},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$20\r\n-9223372036854775808\r\n",
},
{
[]interface{}{"SET", "key", float64(1349673917.939762)},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$21\r\n1.349673917939762e+09\r\n",
},
{
[]interface{}{"SET", "key", ""},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$0\r\n\r\n",
},
{
[]interface{}{"SET", "key", nil},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$0\r\n\r\n",
},
{
[]interface{}{"SET", "key", durationArg{time.Minute}},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$2\r\n60\r\n",
},
{
[]interface{}{"SET", "key", recursiveArg(123)},
"*3\r\n$3\r\nSET\r\n$3\r\nkey\r\n$3\r\n123\r\n",
},
{
[]interface{}{"ECHO", true, false},
"*3\r\n$4\r\nECHO\r\n$1\r\n1\r\n$1\r\n0\r\n",
},
}
func TestWrite(t *testing.T) {
for _, tt := range writeTests {
var buf bytes.Buffer
c, _ := redis.Dial("", "", dialTestConn("", &buf))
err := c.Send(tt.args[0].(string), tt.args[1:]...)
if err != nil {
t.Errorf("Send(%v) returned error %v", tt.args, err)
continue
}
c.Flush()
actual := buf.String()
if actual != tt.expected {
t.Errorf("Send(%v) = %q, want %q", tt.args, actual, tt.expected)
}
}
}
var errorSentinel = &struct{}{}
var readTests = []struct {
reply string
expected interface{}
}{
{
"+OK\r\n",
"OK",
},
{
"+PONG\r\n",
"PONG",
},
{
"@OK\r\n",
errorSentinel,
},
{
"$6\r\nfoobar\r\n",
[]byte("foobar"),
},
{
"$-1\r\n",
nil,
},
{
":1\r\n",
int64(1),
},
{
":-2\r\n",
int64(-2),
},
{
"*0\r\n",
[]interface{}{},
},
{
"*-1\r\n",
nil,
},
{
"*4\r\n$3\r\nfoo\r\n$3\r\nbar\r\n$5\r\nHello\r\n$5\r\nWorld\r\n",
[]interface{}{[]byte("foo"), []byte("bar"), []byte("Hello"), []byte("World")},
},
{
"*3\r\n$3\r\nfoo\r\n$-1\r\n$3\r\nbar\r\n",
[]interface{}{[]byte("foo"), nil, []byte("bar")},
},
{
// "x" is not a valid length
"$x\r\nfoobar\r\n",
errorSentinel,
},
{
// -2 is not a valid length
"$-2\r\n",
errorSentinel,
},
{
// "x" is not a valid integer
":x\r\n",
errorSentinel,
},
{
// missing \r\n following value
"$6\r\nfoobar",
errorSentinel,
},
{
// short value
"$6\r\nxx",
errorSentinel,
},
{
// long value
"$6\r\nfoobarx\r\n",
errorSentinel,
},
}
func TestRead(t *testing.T) {
for _, tt := range readTests {
c, _ := redis.Dial("", "", dialTestConn(tt.reply, nil))
actual, err := c.Receive()
if tt.expected == errorSentinel {
if err == nil {
t.Errorf("Receive(%q) did not return expected error", tt.reply)
}
} else {
if err != nil {
t.Errorf("Receive(%q) returned error %v", tt.reply, err)
continue
}
if !reflect.DeepEqual(actual, tt.expected) {
t.Errorf("Receive(%q) = %v, want %v", tt.reply, actual, tt.expected)
}
}
}
}
var testCommands = []struct {
args []interface{}
expected interface{}
}{
{
[]interface{}{"PING"},
"PONG",
},
{
[]interface{}{"SET", "foo", "bar"},
"OK",
},
{
[]interface{}{"GET", "foo"},
[]byte("bar"),
},
{
[]interface{}{"GET", "nokey"},
nil,
},
{
[]interface{}{"MGET", "nokey", "foo"},
[]interface{}{nil, []byte("bar")},
},
{
[]interface{}{"INCR", "mycounter"},
int64(1),
},
{
[]interface{}{"LPUSH", "mylist", "foo"},
int64(1),
},
{
[]interface{}{"LPUSH", "mylist", "bar"},
int64(2),
},
{
[]interface{}{"LRANGE", "mylist", 0, -1},
[]interface{}{[]byte("bar"), []byte("foo")},
},
{
[]interface{}{"MULTI"},
"OK",
},
{
[]interface{}{"LRANGE", "mylist", 0, -1},
"QUEUED",
},
{
[]interface{}{"PING"},
"QUEUED",
},
{
[]interface{}{"EXEC"},
[]interface{}{
[]interface{}{[]byte("bar"), []byte("foo")},
"PONG",
},
},
}
func TestDoCommands(t *testing.T) {
c, err := redis.DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer c.Close()
for _, cmd := range testCommands {
actual, err := c.Do(cmd.args[0].(string), cmd.args[1:]...)
if err != nil {
t.Errorf("Do(%v) returned error %v", cmd.args, err)
continue
}
if !reflect.DeepEqual(actual, cmd.expected) {
t.Errorf("Do(%v) = %v, want %v", cmd.args, actual, cmd.expected)
}
}
}
func TestPipelineCommands(t *testing.T) {
c, err := redis.DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer c.Close()
for _, cmd := range testCommands {
if err := c.Send(cmd.args[0].(string), cmd.args[1:]...); err != nil {
t.Fatalf("Send(%v) returned error %v", cmd.args, err)
}
}
if err := c.Flush(); err != nil {
t.Errorf("Flush() returned error %v", err)
}
for _, cmd := range testCommands {
actual, err := c.Receive()
if err != nil {
t.Fatalf("Receive(%v) returned error %v", cmd.args, err)
}
if !reflect.DeepEqual(actual, cmd.expected) {
t.Errorf("Receive(%v) = %v, want %v", cmd.args, actual, cmd.expected)
}
}
}
func TestBlankCommmand(t *testing.T) {
c, err := redis.DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer c.Close()
for _, cmd := range testCommands {
if err := c.Send(cmd.args[0].(string), cmd.args[1:]...); err != nil {
t.Fatalf("Send(%v) returned error %v", cmd.args, err)
}
}
reply, err := redis.Values(c.Do(""))
if err != nil {
t.Fatalf("Do() returned error %v", err)
}
if len(reply) != len(testCommands) {
t.Fatalf("len(reply)=%d, want %d", len(reply), len(testCommands))
}
for i, cmd := range testCommands {
actual := reply[i]
if !reflect.DeepEqual(actual, cmd.expected) {
t.Errorf("Receive(%v) = %v, want %v", cmd.args, actual, cmd.expected)
}
}
}
func TestRecvBeforeSend(t *testing.T) {
c, err := redis.DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer c.Close()
done := make(chan struct{})
go func() {
c.Receive()
close(done)
}()
time.Sleep(time.Millisecond)
c.Send("PING")
c.Flush()
<-done
_, err = c.Do("")
if err != nil {
t.Fatalf("error=%v", err)
}
}
func TestError(t *testing.T) {
c, err := redis.DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer c.Close()
c.Do("SET", "key", "val")
_, err = c.Do("HSET", "key", "fld", "val")
if err == nil {
t.Errorf("Expected err for HSET on string key.")
}
if c.Err() != nil {
t.Errorf("Conn has Err()=%v, expect nil", c.Err())
}
_, err = c.Do("SET", "key", "val")
if err != nil {
t.Errorf("Do(SET, key, val) returned error %v, expected nil.", err)
}
}
func TestReadTimeout(t *testing.T) {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("net.Listen returned %v", err)
}
defer l.Close()
go func() {
for {
c, err := l.Accept()
if err != nil {
return
}
go func() {
time.Sleep(time.Second)
c.Write([]byte("+OK\r\n"))
c.Close()
}()
}
}()
// Do
c1, err := redis.Dial(l.Addr().Network(), l.Addr().String(), redis.DialReadTimeout(time.Millisecond))
if err != nil {
t.Fatalf("redis.Dial returned %v", err)
}
defer c1.Close()
_, err = c1.Do("PING")
if err == nil {
t.Fatalf("c1.Do() returned nil, expect error")
}
if c1.Err() == nil {
t.Fatalf("c1.Err() = nil, expect error")
}
// Send/Flush/Receive
c2, err := redis.Dial(l.Addr().Network(), l.Addr().String(), redis.DialReadTimeout(time.Millisecond))
if err != nil {
t.Fatalf("redis.Dial returned %v", err)
}
defer c2.Close()
c2.Send("PING")
c2.Flush()
_, err = c2.Receive()
if err == nil {
t.Fatalf("c2.Receive() returned nil, expect error")
}
if c2.Err() == nil {
t.Fatalf("c2.Err() = nil, expect error")
}
}
var dialErrors = []struct {
rawurl string
expectedError string
}{
{
"localhost",
"invalid redis URL scheme",
},
// The error message for invalid hosts is different in different
// versions of Go, so just check that there is an error message.
{
"redis://weird url",
"",
},
{
"redis://foo:bar:baz",
"",
},
{
"http://www.google.com",
"invalid redis URL scheme: http",
},
{
"redis://localhost:6379/abc123",
"invalid database: abc123",
},
}
func TestDialURLErrors(t *testing.T) {
for _, d := range dialErrors {
_, err := redis.DialURL(d.rawurl)
if err == nil || !strings.Contains(err.Error(), d.expectedError) {
t.Errorf("DialURL did not return expected error (expected %v to contain %s)", err, d.expectedError)
}
}
}
func TestDialURLPort(t *testing.T) {
checkPort := func(network, address string) (net.Conn, error) {
if address != "localhost:6379" {
t.Errorf("DialURL did not set port to 6379 by default (got %v)", address)
}
return nil, nil
}
_, err := redis.DialURL("redis://localhost", redis.DialNetDial(checkPort))
if err != nil {
t.Error("dial error:", err)
}
}
func TestDialURLHost(t *testing.T) {
checkHost := func(network, address string) (net.Conn, error) {
if address != "localhost:6379" {
t.Errorf("DialURL did not set host to localhost by default (got %v)", address)
}
return nil, nil
}
_, err := redis.DialURL("redis://:6379", redis.DialNetDial(checkHost))
if err != nil {
t.Error("dial error:", err)
}
}
var dialURLTests = []struct {
description string
url string
r string
w string
}{
{"password", "redis://x:abc123@localhost", "+OK\r\n", "*2\r\n$4\r\nAUTH\r\n$6\r\nabc123\r\n"},
{"database 3", "redis://localhost/3", "+OK\r\n", "*2\r\n$6\r\nSELECT\r\n$1\r\n3\r\n"},
{"database 99", "redis://localhost/99", "+OK\r\n", "*2\r\n$6\r\nSELECT\r\n$2\r\n99\r\n"},
{"no database", "redis://localhost/", "+OK\r\n", ""},
}
func TestDialURL(t *testing.T) {
for _, tt := range dialURLTests {
var buf bytes.Buffer
// UseTLS should be ignored in all of these tests.
_, err := redis.DialURL(tt.url, dialTestConn(tt.r, &buf), redis.DialUseTLS(true))
if err != nil {
t.Errorf("%s dial error: %v", tt.description, err)
continue
}
if w := buf.String(); w != tt.w {
t.Errorf("%s commands = %q, want %q", tt.description, w, tt.w)
}
}
}
func checkPingPong(t *testing.T, buf *bytes.Buffer, c redis.Conn) {
resp, err := c.Do("PING")
if err != nil {
t.Fatal("ping error:", err)
}
// Close connection to ensure that writes to buf are complete.
c.Close()
expected := "*1\r\n$4\r\nPING\r\n"
actual := buf.String()
if actual != expected {
t.Errorf("commands = %q, want %q", actual, expected)
}
if resp != "PONG" {
t.Errorf("resp = %v, want %v", resp, "PONG")
}
}
const pingResponse = "+PONG\r\n"
func TestDialURLTLS(t *testing.T) {
var buf bytes.Buffer
c, err := redis.DialURL("rediss://example.com/",
redis.DialTLSConfig(&clientTLSConfig),
dialTestConnTLS(pingResponse, &buf))
if err != nil {
t.Fatal("dial error:", err)
}
checkPingPong(t, &buf, c)
}
func TestDialUseTLS(t *testing.T) {
var buf bytes.Buffer
c, err := redis.Dial("tcp", "example.com:6379",
redis.DialTLSConfig(&clientTLSConfig),
dialTestConnTLS(pingResponse, &buf),
redis.DialUseTLS(true))
if err != nil {
t.Fatal("dial error:", err)
}
checkPingPong(t, &buf, c)
}
func TestDialTLSSKipVerify(t *testing.T) {
var buf bytes.Buffer
c, err := redis.Dial("tcp", "example.com:6379",
dialTestConnTLS(pingResponse, &buf),
redis.DialTLSSkipVerify(true),
redis.DialUseTLS(true))
if err != nil {
t.Fatal("dial error:", err)
}
checkPingPong(t, &buf, c)
}
// Connect to local instance of Redis running on the default port.
func ExampleDial() {
c, err := redis.Dial("tcp", ":6379")
if err != nil {
// handle error
}
defer c.Close()
}
// Connect to remote instance of Redis using a URL.
func ExampleDialURL() {
c, err := redis.DialURL(os.Getenv("REDIS_URL"))
if err != nil {
// handle connection error
}
defer c.Close()
}
// TextExecError tests handling of errors in a transaction. See
// http://redis.io/topics/transactions for information on how Redis handles
// errors in a transaction.
func TestExecError(t *testing.T) {
c, err := redis.DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer c.Close()
// Execute commands that fail before EXEC is called.
c.Do("DEL", "k0")
c.Do("ZADD", "k0", 0, 0)
c.Send("MULTI")
c.Send("NOTACOMMAND", "k0", 0, 0)
c.Send("ZINCRBY", "k0", 0, 0)
v, err := c.Do("EXEC")
if err == nil {
t.Fatalf("EXEC returned values %v, expected error", v)
}
// Execute commands that fail after EXEC is called. The first command
// returns an error.
c.Do("DEL", "k1")
c.Do("ZADD", "k1", 0, 0)
c.Send("MULTI")
c.Send("HSET", "k1", 0, 0)
c.Send("ZINCRBY", "k1", 0, 0)
v, err = c.Do("EXEC")
if err != nil {
t.Fatalf("EXEC returned error %v", err)
}
vs, err := redis.Values(v, nil)
if err != nil {
t.Fatalf("Values(v) returned error %v", err)
}
if len(vs) != 2 {
t.Fatalf("len(vs) == %d, want 2", len(vs))
}
if _, ok := vs[0].(error); !ok {
t.Fatalf("first result is type %T, expected error", vs[0])
}
if _, ok := vs[1].([]byte); !ok {
t.Fatalf("second result is type %T, expected []byte", vs[1])
}
// Execute commands that fail after EXEC is called. The second command
// returns an error.
c.Do("ZADD", "k2", 0, 0)
c.Send("MULTI")
c.Send("ZINCRBY", "k2", 0, 0)
c.Send("HSET", "k2", 0, 0)
v, err = c.Do("EXEC")
if err != nil {
t.Fatalf("EXEC returned error %v", err)
}
vs, err = redis.Values(v, nil)
if err != nil {
t.Fatalf("Values(v) returned error %v", err)
}
if len(vs) != 2 {
t.Fatalf("len(vs) == %d, want 2", len(vs))
}
if _, ok := vs[0].([]byte); !ok {
t.Fatalf("first result is type %T, expected []byte", vs[0])
}
if _, ok := vs[1].(error); !ok {
t.Fatalf("second result is type %T, expected error", vs[2])
}
}
func BenchmarkDoEmpty(b *testing.B) {
b.StopTimer()
c, err := redis.DialDefaultServer()
if err != nil {
b.Fatal(err)
}
defer c.Close()
b.StartTimer()
for i := 0; i < b.N; i++ {
if _, err := c.Do(""); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkDoPing(b *testing.B) {
b.StopTimer()
c, err := redis.DialDefaultServer()
if err != nil {
b.Fatal(err)
}
defer c.Close()
b.StartTimer()
for i := 0; i < b.N; i++ {
if _, err := c.Do("PING"); err != nil {
b.Fatal(err)
}
}
}
var clientTLSConfig, serverTLSConfig tls.Config
func init() {
// The certificate and key for testing TLS dial options was created
// using the command
//
// go run GOROOT/src/crypto/tls/generate_cert.go \
// --rsa-bits 1024 \
// --host 127.0.0.1,::1,example.com --ca \
// --start-date "Jan 1 00:00:00 1970" \
// --duration=1000000h
//
// where GOROOT is the value of GOROOT reported by go env.
localhostCert := []byte(`
-----BEGIN CERTIFICATE-----
MIICFDCCAX2gAwIBAgIRAJfBL4CUxkXcdlFurb3K+iowDQYJKoZIhvcNAQELBQAw
EjEQMA4GA1UEChMHQWNtZSBDbzAgFw03MDAxMDEwMDAwMDBaGA8yMDg0MDEyOTE2
MDAwMFowEjEQMA4GA1UEChMHQWNtZSBDbzCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
gYkCgYEArizw8WxMUQ3bGHLeuJ4fDrEpy+L2pqrbYRlKk1DasJ/VkB8bImzIpe6+
LGjiYIxvnDCOJ3f3QplcQuiuMyl6f2irJlJsbFT8Lo/3obnuTKAIaqUdJUqBg6y+
JaL8Auk97FvunfKFv8U1AIhgiLzAfQ/3Eaq1yi87Ra6pMjGbTtcCAwEAAaNoMGYw
DgYDVR0PAQH/BAQDAgKkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQF
MAMBAf8wLgYDVR0RBCcwJYILZXhhbXBsZS5jb22HBH8AAAGHEAAAAAAAAAAAAAAA
AAAAAAEwDQYJKoZIhvcNAQELBQADgYEAdZ8daIVkyhVwflt5I19m0oq1TycbGO1+
ach7T6cZiBQeNR/SJtxr/wKPEpmvUgbv2BfFrKJ8QoIHYsbNSURTWSEa02pfw4k9
6RQhij3ZkG79Ituj5OYRORV6Z0HUW32r670BtcuHuAhq7YA6Nxy4FtSt7bAlVdRt
rrKgNsltzMk=
-----END CERTIFICATE-----`)
localhostKey := []byte(`
-----BEGIN RSA PRIVATE KEY-----
MIICXAIBAAKBgQCuLPDxbExRDdsYct64nh8OsSnL4vamqtthGUqTUNqwn9WQHxsi
bMil7r4saOJgjG+cMI4nd/dCmVxC6K4zKXp/aKsmUmxsVPwuj/ehue5MoAhqpR0l
SoGDrL4lovwC6T3sW+6d8oW/xTUAiGCIvMB9D/cRqrXKLztFrqkyMZtO1wIDAQAB
AoGACrc5G6FOEK6JjDeE/Fa+EmlT6PdNtXNNi+vCas3Opo8u1G8VfEi1D4BgstrB
Eq+RLkrOdB8tVyuYQYWPMhabMqF+hhKJN72j0OwfuPlVvTInwb/cKjo/zbH1IA+Y
HenHNK4ywv7/p/9/MvQPJ3I32cQBCgGUW5chVSH5M1sj5gECQQDabQAI1X0uDqCm
KbX9gXVkAgxkFddrt6LBHt57xujFcqEKFE7nwKhDh7DweVs/VEJ+kpid4z+UnLOw
KjtP9JolAkEAzCNBphQ//IsbH5rNs10wIUw3Ks/Oepicvr6kUFbIv+neRzi1iJHa
m6H7EayK3PWgax6BAsR/t0Jc9XV7r2muSwJAVzN09BHnK+ADGtNEKLTqXMbEk6B0
pDhn7ZmZUOkUPN+Kky+QYM11X6Bob1jDqQDGmymDbGUxGO+GfSofC8inUQJAGfci
Eo3g1a6b9JksMPRZeuLG4ZstGErxJRH6tH1Va5PDwitka8qhk8o2tTjNMO3NSdLH
diKoXBcE2/Pll5pJoQJBAIMiiMIzXJhnN4mX8may44J/HvMlMf2xuVH2gNMwmZuc
Bjqn3yoLHaoZVvbWOi0C2TCN4FjXjaLNZGifQPbIcaA=
-----END RSA PRIVATE KEY-----`)
cert, err := tls.X509KeyPair(localhostCert, localhostKey)
if err != nil {
panic(fmt.Sprintf("error creating key pair: %v", err))
}
serverTLSConfig.Certificates = []tls.Certificate{cert}
certificate, err := x509.ParseCertificate(serverTLSConfig.Certificates[0].Certificate[0])
if err != nil {
panic(fmt.Sprintf("error parsing x509 certificate: %v", err))
}
clientTLSConfig.RootCAs = x509.NewCertPool()
clientTLSConfig.RootCAs.AddCert(certificate)
}
func TestWithTimeout(t *testing.T) {
for _, recv := range []bool{true, false} {
for _, defaultTimout := range []time.Duration{0, time.Minute} {
var buf bytes.Buffer
nc := &testConn{Reader: strings.NewReader("+OK\r\n+OK\r\n+OK\r\n+OK\r\n+OK\r\n+OK\r\n+OK\r\n+OK\r\n+OK\r\n+OK\r\n"), Writer: &buf}
c, _ := redis.Dial("", "", redis.DialReadTimeout(defaultTimout), redis.DialNetDial(func(network, addr string) (net.Conn, error) { return nc, nil }))
for i := 0; i < 4; i++ {
var minDeadline, maxDeadline time.Time
// Alternate between default and specified timeout.
if i%2 == 0 {
if defaultTimout != 0 {
minDeadline = time.Now().Add(defaultTimout)
}
if recv {
c.Receive()
} else {
c.Do("PING")
}
if defaultTimout != 0 {
maxDeadline = time.Now().Add(defaultTimout)
}
} else {
timeout := 10 * time.Minute
minDeadline = time.Now().Add(timeout)
if recv {
redis.ReceiveWithTimeout(c, timeout)
} else {
redis.DoWithTimeout(c, timeout, "PING")
}
maxDeadline = time.Now().Add(timeout)
}
// Expect set deadline in expected range.
if nc.readDeadline.Before(minDeadline) || nc.readDeadline.After(maxDeadline) {
t.Errorf("recv %v, %d: do deadline error: %v, %v, %v", recv, i, minDeadline, nc.readDeadline, maxDeadline)
}
}
}
}
}

177
src/vendor/github.com/gomodule/redigo/redis/doc.go generated vendored Normal file
View File

@ -0,0 +1,177 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
// Package redis is a client for the Redis database.
//
// The Redigo FAQ (https://github.com/gomodule/redigo/wiki/FAQ) contains more
// documentation about this package.
//
// Connections
//
// The Conn interface is the primary interface for working with Redis.
// Applications create connections by calling the Dial, DialWithTimeout or
// NewConn functions. In the future, functions will be added for creating
// sharded and other types of connections.
//
// The application must call the connection Close method when the application
// is done with the connection.
//
// Executing Commands
//
// The Conn interface has a generic method for executing Redis commands:
//
// Do(commandName string, args ...interface{}) (reply interface{}, err error)
//
// The Redis command reference (http://redis.io/commands) lists the available
// commands. An example of using the Redis APPEND command is:
//
// n, err := conn.Do("APPEND", "key", "value")
//
// The Do method converts command arguments to bulk strings for transmission
// to the server as follows:
//
// Go Type Conversion
// []byte Sent as is
// string Sent as is
// int, int64 strconv.FormatInt(v)
// float64 strconv.FormatFloat(v, 'g', -1, 64)
// bool true -> "1", false -> "0"
// nil ""
// all other types fmt.Fprint(w, v)
//
// Redis command reply types are represented using the following Go types:
//
// Redis type Go type
// error redis.Error
// integer int64
// simple string string
// bulk string []byte or nil if value not present.
// array []interface{} or nil if value not present.
//
// Use type assertions or the reply helper functions to convert from
// interface{} to the specific Go type for the command result.
//
// Pipelining
//
// Connections support pipelining using the Send, Flush and Receive methods.
//
// Send(commandName string, args ...interface{}) error
// Flush() error
// Receive() (reply interface{}, err error)
//
// Send writes the command to the connection's output buffer. Flush flushes the
// connection's output buffer to the server. Receive reads a single reply from
// the server. The following example shows a simple pipeline.
//
// c.Send("SET", "foo", "bar")
// c.Send("GET", "foo")
// c.Flush()
// c.Receive() // reply from SET
// v, err = c.Receive() // reply from GET
//
// The Do method combines the functionality of the Send, Flush and Receive
// methods. The Do method starts by writing the command and flushing the output
// buffer. Next, the Do method receives all pending replies including the reply
// for the command just sent by Do. If any of the received replies is an error,
// then Do returns the error. If there are no errors, then Do returns the last
// reply. If the command argument to the Do method is "", then the Do method
// will flush the output buffer and receive pending replies without sending a
// command.
//
// Use the Send and Do methods to implement pipelined transactions.
//
// c.Send("MULTI")
// c.Send("INCR", "foo")
// c.Send("INCR", "bar")
// r, err := c.Do("EXEC")
// fmt.Println(r) // prints [1, 1]
//
// Concurrency
//
// Connections support one concurrent caller to the Receive method and one
// concurrent caller to the Send and Flush methods. No other concurrency is
// supported including concurrent calls to the Do method.
//
// For full concurrent access to Redis, use the thread-safe Pool to get, use
// and release a connection from within a goroutine. Connections returned from
// a Pool have the concurrency restrictions described in the previous
// paragraph.
//
// Publish and Subscribe
//
// Use the Send, Flush and Receive methods to implement Pub/Sub subscribers.
//
// c.Send("SUBSCRIBE", "example")
// c.Flush()
// for {
// reply, err := c.Receive()
// if err != nil {
// return err
// }
// // process pushed message
// }
//
// The PubSubConn type wraps a Conn with convenience methods for implementing
// subscribers. The Subscribe, PSubscribe, Unsubscribe and PUnsubscribe methods
// send and flush a subscription management command. The receive method
// converts a pushed message to convenient types for use in a type switch.
//
// psc := redis.PubSubConn{Conn: c}
// psc.Subscribe("example")
// for {
// switch v := psc.Receive().(type) {
// case redis.Message:
// fmt.Printf("%s: message: %s\n", v.Channel, v.Data)
// case redis.Subscription:
// fmt.Printf("%s: %s %d\n", v.Channel, v.Kind, v.Count)
// case error:
// return v
// }
// }
//
// Reply Helpers
//
// The Bool, Int, Bytes, String, Strings and Values functions convert a reply
// to a value of a specific type. To allow convenient wrapping of calls to the
// connection Do and Receive methods, the functions take a second argument of
// type error. If the error is non-nil, then the helper function returns the
// error. If the error is nil, the function converts the reply to the specified
// type:
//
// exists, err := redis.Bool(c.Do("EXISTS", "foo"))
// if err != nil {
// // handle error return from c.Do or type conversion error.
// }
//
// The Scan function converts elements of a array reply to Go types:
//
// var value1 int
// var value2 string
// reply, err := redis.Values(c.Do("MGET", "key1", "key2"))
// if err != nil {
// // handle error
// }
// if _, err := redis.Scan(reply, &value1, &value2); err != nil {
// // handle error
// }
//
// Errors
//
// Connection methods return error replies from the server as type redis.Error.
//
// Call the connection Err() method to determine if the connection encountered
// non-recoverable error such as a network error or protocol parsing error. If
// Err() returns a non-nil value, then the connection is not usable and should
// be closed.
package redis // import "github.com/gomodule/redigo/redis"

27
src/vendor/github.com/gomodule/redigo/redis/go16.go generated vendored Normal file
View File

@ -0,0 +1,27 @@
// +build !go1.7
package redis
import "crypto/tls"
func cloneTLSConfig(cfg *tls.Config) *tls.Config {
return &tls.Config{
Rand: cfg.Rand,
Time: cfg.Time,
Certificates: cfg.Certificates,
NameToCertificate: cfg.NameToCertificate,
GetCertificate: cfg.GetCertificate,
RootCAs: cfg.RootCAs,
NextProtos: cfg.NextProtos,
ServerName: cfg.ServerName,
ClientAuth: cfg.ClientAuth,
ClientCAs: cfg.ClientCAs,
InsecureSkipVerify: cfg.InsecureSkipVerify,
CipherSuites: cfg.CipherSuites,
PreferServerCipherSuites: cfg.PreferServerCipherSuites,
ClientSessionCache: cfg.ClientSessionCache,
MinVersion: cfg.MinVersion,
MaxVersion: cfg.MaxVersion,
CurvePreferences: cfg.CurvePreferences,
}
}

29
src/vendor/github.com/gomodule/redigo/redis/go17.go generated vendored Normal file
View File

@ -0,0 +1,29 @@
// +build go1.7,!go1.8
package redis
import "crypto/tls"
func cloneTLSConfig(cfg *tls.Config) *tls.Config {
return &tls.Config{
Rand: cfg.Rand,
Time: cfg.Time,
Certificates: cfg.Certificates,
NameToCertificate: cfg.NameToCertificate,
GetCertificate: cfg.GetCertificate,
RootCAs: cfg.RootCAs,
NextProtos: cfg.NextProtos,
ServerName: cfg.ServerName,
ClientAuth: cfg.ClientAuth,
ClientCAs: cfg.ClientCAs,
InsecureSkipVerify: cfg.InsecureSkipVerify,
CipherSuites: cfg.CipherSuites,
PreferServerCipherSuites: cfg.PreferServerCipherSuites,
ClientSessionCache: cfg.ClientSessionCache,
MinVersion: cfg.MinVersion,
MaxVersion: cfg.MaxVersion,
CurvePreferences: cfg.CurvePreferences,
DynamicRecordSizingDisabled: cfg.DynamicRecordSizingDisabled,
Renegotiation: cfg.Renegotiation,
}
}

9
src/vendor/github.com/gomodule/redigo/redis/go18.go generated vendored Normal file
View File

@ -0,0 +1,9 @@
// +build go1.8
package redis
import "crypto/tls"
func cloneTLSConfig(cfg *tls.Config) *tls.Config {
return cfg.Clone()
}

View File

@ -0,0 +1,85 @@
// Copyright 2018 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
// +build go1.9
package redis
import "testing"
func TestPoolList(t *testing.T) {
var idle idleList
var a, b, c poolConn
check := func(pcs ...*poolConn) {
if idle.count != len(pcs) {
t.Fatal("idle.count != len(pcs)")
}
if len(pcs) == 0 {
if idle.front != nil {
t.Fatalf("front not nil")
}
if idle.back != nil {
t.Fatalf("back not nil")
}
return
}
if idle.front != pcs[0] {
t.Fatal("front != pcs[0]")
}
if idle.back != pcs[len(pcs)-1] {
t.Fatal("back != pcs[len(pcs)-1]")
}
if idle.front.prev != nil {
t.Fatal("front.prev != nil")
}
if idle.back.next != nil {
t.Fatal("back.next != nil")
}
for i := 1; i < len(pcs)-1; i++ {
if pcs[i-1].next != pcs[i] {
t.Fatal("pcs[i-1].next != pcs[i]")
}
if pcs[i+1].prev != pcs[i] {
t.Fatal("pcs[i+1].prev != pcs[i]")
}
}
}
idle.pushFront(&c)
check(&c)
idle.pushFront(&b)
check(&b, &c)
idle.pushFront(&a)
check(&a, &b, &c)
idle.popFront()
check(&b, &c)
idle.popFront()
check(&c)
idle.popFront()
check()
idle.pushFront(&c)
check(&c)
idle.pushFront(&b)
check(&b, &c)
idle.pushFront(&a)
check(&a, &b, &c)
idle.popBack()
check(&a, &b)
idle.popBack()
check(&a)
idle.popBack()
check()
}

134
src/vendor/github.com/gomodule/redigo/redis/log.go generated vendored Normal file
View File

@ -0,0 +1,134 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis
import (
"bytes"
"fmt"
"log"
"time"
)
var (
_ ConnWithTimeout = (*loggingConn)(nil)
)
// NewLoggingConn returns a logging wrapper around a connection.
func NewLoggingConn(conn Conn, logger *log.Logger, prefix string) Conn {
if prefix != "" {
prefix = prefix + "."
}
return &loggingConn{conn, logger, prefix}
}
type loggingConn struct {
Conn
logger *log.Logger
prefix string
}
func (c *loggingConn) Close() error {
err := c.Conn.Close()
var buf bytes.Buffer
fmt.Fprintf(&buf, "%sClose() -> (%v)", c.prefix, err)
c.logger.Output(2, buf.String())
return err
}
func (c *loggingConn) printValue(buf *bytes.Buffer, v interface{}) {
const chop = 32
switch v := v.(type) {
case []byte:
if len(v) > chop {
fmt.Fprintf(buf, "%q...", v[:chop])
} else {
fmt.Fprintf(buf, "%q", v)
}
case string:
if len(v) > chop {
fmt.Fprintf(buf, "%q...", v[:chop])
} else {
fmt.Fprintf(buf, "%q", v)
}
case []interface{}:
if len(v) == 0 {
buf.WriteString("[]")
} else {
sep := "["
fin := "]"
if len(v) > chop {
v = v[:chop]
fin = "...]"
}
for _, vv := range v {
buf.WriteString(sep)
c.printValue(buf, vv)
sep = ", "
}
buf.WriteString(fin)
}
default:
fmt.Fprint(buf, v)
}
}
func (c *loggingConn) print(method, commandName string, args []interface{}, reply interface{}, err error) {
var buf bytes.Buffer
fmt.Fprintf(&buf, "%s%s(", c.prefix, method)
if method != "Receive" {
buf.WriteString(commandName)
for _, arg := range args {
buf.WriteString(", ")
c.printValue(&buf, arg)
}
}
buf.WriteString(") -> (")
if method != "Send" {
c.printValue(&buf, reply)
buf.WriteString(", ")
}
fmt.Fprintf(&buf, "%v)", err)
c.logger.Output(3, buf.String())
}
func (c *loggingConn) Do(commandName string, args ...interface{}) (interface{}, error) {
reply, err := c.Conn.Do(commandName, args...)
c.print("Do", commandName, args, reply, err)
return reply, err
}
func (c *loggingConn) DoWithTimeout(timeout time.Duration, commandName string, args ...interface{}) (interface{}, error) {
reply, err := DoWithTimeout(c.Conn, timeout, commandName, args...)
c.print("DoWithTimeout", commandName, args, reply, err)
return reply, err
}
func (c *loggingConn) Send(commandName string, args ...interface{}) error {
err := c.Conn.Send(commandName, args...)
c.print("Send", commandName, args, nil, err)
return err
}
func (c *loggingConn) Receive() (interface{}, error) {
reply, err := c.Conn.Receive()
c.print("Receive", "", nil, reply, err)
return reply, err
}
func (c *loggingConn) ReceiveWithTimeout(timeout time.Duration) (interface{}, error) {
reply, err := ReceiveWithTimeout(c.Conn, timeout)
c.print("ReceiveWithTimeout", "", nil, reply, err)
return reply, err
}

562
src/vendor/github.com/gomodule/redigo/redis/pool.go generated vendored Normal file
View File

@ -0,0 +1,562 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis
import (
"bytes"
"crypto/rand"
"crypto/sha1"
"errors"
"io"
"strconv"
"sync"
"sync/atomic"
"time"
"github.com/gomodule/redigo/internal"
)
var (
_ ConnWithTimeout = (*activeConn)(nil)
_ ConnWithTimeout = (*errorConn)(nil)
)
var nowFunc = time.Now // for testing
// ErrPoolExhausted is returned from a pool connection method (Do, Send,
// Receive, Flush, Err) when the maximum number of database connections in the
// pool has been reached.
var ErrPoolExhausted = errors.New("redigo: connection pool exhausted")
var (
errPoolClosed = errors.New("redigo: connection pool closed")
errConnClosed = errors.New("redigo: connection closed")
)
// Pool maintains a pool of connections. The application calls the Get method
// to get a connection from the pool and the connection's Close method to
// return the connection's resources to the pool.
//
// The following example shows how to use a pool in a web application. The
// application creates a pool at application startup and makes it available to
// request handlers using a package level variable. The pool configuration used
// here is an example, not a recommendation.
//
// func newPool(addr string) *redis.Pool {
// return &redis.Pool{
// MaxIdle: 3,
// IdleTimeout: 240 * time.Second,
// Dial: func () (redis.Conn, error) { return redis.Dial("tcp", addr) },
// }
// }
//
// var (
// pool *redis.Pool
// redisServer = flag.String("redisServer", ":6379", "")
// )
//
// func main() {
// flag.Parse()
// pool = newPool(*redisServer)
// ...
// }
//
// A request handler gets a connection from the pool and closes the connection
// when the handler is done:
//
// func serveHome(w http.ResponseWriter, r *http.Request) {
// conn := pool.Get()
// defer conn.Close()
// ...
// }
//
// Use the Dial function to authenticate connections with the AUTH command or
// select a database with the SELECT command:
//
// pool := &redis.Pool{
// // Other pool configuration not shown in this example.
// Dial: func () (redis.Conn, error) {
// c, err := redis.Dial("tcp", server)
// if err != nil {
// return nil, err
// }
// if _, err := c.Do("AUTH", password); err != nil {
// c.Close()
// return nil, err
// }
// if _, err := c.Do("SELECT", db); err != nil {
// c.Close()
// return nil, err
// }
// return c, nil
// },
// }
//
// Use the TestOnBorrow function to check the health of an idle connection
// before the connection is returned to the application. This example PINGs
// connections that have been idle more than a minute:
//
// pool := &redis.Pool{
// // Other pool configuration not shown in this example.
// TestOnBorrow: func(c redis.Conn, t time.Time) error {
// if time.Since(t) < time.Minute {
// return nil
// }
// _, err := c.Do("PING")
// return err
// },
// }
//
type Pool struct {
// Dial is an application supplied function for creating and configuring a
// connection.
//
// The connection returned from Dial must not be in a special state
// (subscribed to pubsub channel, transaction started, ...).
Dial func() (Conn, error)
// TestOnBorrow is an optional application supplied function for checking
// the health of an idle connection before the connection is used again by
// the application. Argument t is the time that the connection was returned
// to the pool. If the function returns an error, then the connection is
// closed.
TestOnBorrow func(c Conn, t time.Time) error
// Maximum number of idle connections in the pool.
MaxIdle int
// Maximum number of connections allocated by the pool at a given time.
// When zero, there is no limit on the number of connections in the pool.
MaxActive int
// Close connections after remaining idle for this duration. If the value
// is zero, then idle connections are not closed. Applications should set
// the timeout to a value less than the server's timeout.
IdleTimeout time.Duration
// If Wait is true and the pool is at the MaxActive limit, then Get() waits
// for a connection to be returned to the pool before returning.
Wait bool
// Close connections older than this duration. If the value is zero, then
// the pool does not close connections based on age.
MaxConnLifetime time.Duration
chInitialized uint32 // set to 1 when field ch is initialized
mu sync.Mutex // mu protects the following fields
closed bool // set to true when the pool is closed.
active int // the number of open connections in the pool
ch chan struct{} // limits open connections when p.Wait is true
idle idleList // idle connections
}
// NewPool creates a new pool.
//
// Deprecated: Initialize the Pool directory as shown in the example.
func NewPool(newFn func() (Conn, error), maxIdle int) *Pool {
return &Pool{Dial: newFn, MaxIdle: maxIdle}
}
// Get gets a connection. The application must close the returned connection.
// This method always returns a valid connection so that applications can defer
// error handling to the first use of the connection. If there is an error
// getting an underlying connection, then the connection Err, Do, Send, Flush
// and Receive methods return that error.
func (p *Pool) Get() Conn {
pc, err := p.get(nil)
if err != nil {
return errorConn{err}
}
return &activeConn{p: p, pc: pc}
}
// PoolStats contains pool statistics.
type PoolStats struct {
// ActiveCount is the number of connections in the pool. The count includes
// idle connections and connections in use.
ActiveCount int
// IdleCount is the number of idle connections in the pool.
IdleCount int
}
// Stats returns pool's statistics.
func (p *Pool) Stats() PoolStats {
p.mu.Lock()
stats := PoolStats{
ActiveCount: p.active,
IdleCount: p.idle.count,
}
p.mu.Unlock()
return stats
}
// ActiveCount returns the number of connections in the pool. The count
// includes idle connections and connections in use.
func (p *Pool) ActiveCount() int {
p.mu.Lock()
active := p.active
p.mu.Unlock()
return active
}
// IdleCount returns the number of idle connections in the pool.
func (p *Pool) IdleCount() int {
p.mu.Lock()
idle := p.idle.count
p.mu.Unlock()
return idle
}
// Close releases the resources used by the pool.
func (p *Pool) Close() error {
p.mu.Lock()
if p.closed {
p.mu.Unlock()
return nil
}
p.closed = true
p.active -= p.idle.count
pc := p.idle.front
p.idle.count = 0
p.idle.front, p.idle.back = nil, nil
if p.ch != nil {
close(p.ch)
}
p.mu.Unlock()
for ; pc != nil; pc = pc.next {
pc.c.Close()
}
return nil
}
func (p *Pool) lazyInit() {
// Fast path.
if atomic.LoadUint32(&p.chInitialized) == 1 {
return
}
// Slow path.
p.mu.Lock()
if p.chInitialized == 0 {
p.ch = make(chan struct{}, p.MaxActive)
if p.closed {
close(p.ch)
} else {
for i := 0; i < p.MaxActive; i++ {
p.ch <- struct{}{}
}
}
atomic.StoreUint32(&p.chInitialized, 1)
}
p.mu.Unlock()
}
// get prunes stale connections and returns a connection from the idle list or
// creates a new connection.
func (p *Pool) get(ctx interface {
Done() <-chan struct{}
Err() error
}) (*poolConn, error) {
// Handle limit for p.Wait == true.
if p.Wait && p.MaxActive > 0 {
p.lazyInit()
if ctx == nil {
<-p.ch
} else {
select {
case <-p.ch:
case <-ctx.Done():
return nil, ctx.Err()
}
}
}
p.mu.Lock()
// Prune stale connections at the back of the idle list.
if p.IdleTimeout > 0 {
n := p.idle.count
for i := 0; i < n && p.idle.back != nil && p.idle.back.t.Add(p.IdleTimeout).Before(nowFunc()); i++ {
pc := p.idle.back
p.idle.popBack()
p.mu.Unlock()
pc.c.Close()
p.mu.Lock()
p.active--
}
}
// Get idle connection from the front of idle list.
for p.idle.front != nil {
pc := p.idle.front
p.idle.popFront()
p.mu.Unlock()
if (p.TestOnBorrow == nil || p.TestOnBorrow(pc.c, pc.t) == nil) &&
(p.MaxConnLifetime == 0 || nowFunc().Sub(pc.created) < p.MaxConnLifetime) {
return pc, nil
}
pc.c.Close()
p.mu.Lock()
p.active--
}
// Check for pool closed before dialing a new connection.
if p.closed {
p.mu.Unlock()
return nil, errors.New("redigo: get on closed pool")
}
// Handle limit for p.Wait == false.
if !p.Wait && p.MaxActive > 0 && p.active >= p.MaxActive {
p.mu.Unlock()
return nil, ErrPoolExhausted
}
p.active++
p.mu.Unlock()
c, err := p.Dial()
if err != nil {
c = nil
p.mu.Lock()
p.active--
if p.ch != nil && !p.closed {
p.ch <- struct{}{}
}
p.mu.Unlock()
}
return &poolConn{c: c, created: nowFunc()}, err
}
func (p *Pool) put(pc *poolConn, forceClose bool) error {
p.mu.Lock()
if !p.closed && !forceClose {
pc.t = nowFunc()
p.idle.pushFront(pc)
if p.idle.count > p.MaxIdle {
pc = p.idle.back
p.idle.popBack()
} else {
pc = nil
}
}
if pc != nil {
p.mu.Unlock()
pc.c.Close()
p.mu.Lock()
p.active--
}
if p.ch != nil && !p.closed {
p.ch <- struct{}{}
}
p.mu.Unlock()
return nil
}
type activeConn struct {
p *Pool
pc *poolConn
state int
}
var (
sentinel []byte
sentinelOnce sync.Once
)
func initSentinel() {
p := make([]byte, 64)
if _, err := rand.Read(p); err == nil {
sentinel = p
} else {
h := sha1.New()
io.WriteString(h, "Oops, rand failed. Use time instead.")
io.WriteString(h, strconv.FormatInt(time.Now().UnixNano(), 10))
sentinel = h.Sum(nil)
}
}
func (ac *activeConn) Close() error {
pc := ac.pc
if pc == nil {
return nil
}
ac.pc = nil
if ac.state&internal.MultiState != 0 {
pc.c.Send("DISCARD")
ac.state &^= (internal.MultiState | internal.WatchState)
} else if ac.state&internal.WatchState != 0 {
pc.c.Send("UNWATCH")
ac.state &^= internal.WatchState
}
if ac.state&internal.SubscribeState != 0 {
pc.c.Send("UNSUBSCRIBE")
pc.c.Send("PUNSUBSCRIBE")
// To detect the end of the message stream, ask the server to echo
// a sentinel value and read until we see that value.
sentinelOnce.Do(initSentinel)
pc.c.Send("ECHO", sentinel)
pc.c.Flush()
for {
p, err := pc.c.Receive()
if err != nil {
break
}
if p, ok := p.([]byte); ok && bytes.Equal(p, sentinel) {
ac.state &^= internal.SubscribeState
break
}
}
}
pc.c.Do("")
ac.p.put(pc, ac.state != 0 || pc.c.Err() != nil)
return nil
}
func (ac *activeConn) Err() error {
pc := ac.pc
if pc == nil {
return errConnClosed
}
return pc.c.Err()
}
func (ac *activeConn) Do(commandName string, args ...interface{}) (reply interface{}, err error) {
pc := ac.pc
if pc == nil {
return nil, errConnClosed
}
ci := internal.LookupCommandInfo(commandName)
ac.state = (ac.state | ci.Set) &^ ci.Clear
return pc.c.Do(commandName, args...)
}
func (ac *activeConn) DoWithTimeout(timeout time.Duration, commandName string, args ...interface{}) (reply interface{}, err error) {
pc := ac.pc
if pc == nil {
return nil, errConnClosed
}
cwt, ok := pc.c.(ConnWithTimeout)
if !ok {
return nil, errTimeoutNotSupported
}
ci := internal.LookupCommandInfo(commandName)
ac.state = (ac.state | ci.Set) &^ ci.Clear
return cwt.DoWithTimeout(timeout, commandName, args...)
}
func (ac *activeConn) Send(commandName string, args ...interface{}) error {
pc := ac.pc
if pc == nil {
return errConnClosed
}
ci := internal.LookupCommandInfo(commandName)
ac.state = (ac.state | ci.Set) &^ ci.Clear
return pc.c.Send(commandName, args...)
}
func (ac *activeConn) Flush() error {
pc := ac.pc
if pc == nil {
return errConnClosed
}
return pc.c.Flush()
}
func (ac *activeConn) Receive() (reply interface{}, err error) {
pc := ac.pc
if pc == nil {
return nil, errConnClosed
}
return pc.c.Receive()
}
func (ac *activeConn) ReceiveWithTimeout(timeout time.Duration) (reply interface{}, err error) {
pc := ac.pc
if pc == nil {
return nil, errConnClosed
}
cwt, ok := pc.c.(ConnWithTimeout)
if !ok {
return nil, errTimeoutNotSupported
}
return cwt.ReceiveWithTimeout(timeout)
}
type errorConn struct{ err error }
func (ec errorConn) Do(string, ...interface{}) (interface{}, error) { return nil, ec.err }
func (ec errorConn) DoWithTimeout(time.Duration, string, ...interface{}) (interface{}, error) {
return nil, ec.err
}
func (ec errorConn) Send(string, ...interface{}) error { return ec.err }
func (ec errorConn) Err() error { return ec.err }
func (ec errorConn) Close() error { return nil }
func (ec errorConn) Flush() error { return ec.err }
func (ec errorConn) Receive() (interface{}, error) { return nil, ec.err }
func (ec errorConn) ReceiveWithTimeout(time.Duration) (interface{}, error) { return nil, ec.err }
type idleList struct {
count int
front, back *poolConn
}
type poolConn struct {
c Conn
t time.Time
created time.Time
next, prev *poolConn
}
func (l *idleList) pushFront(pc *poolConn) {
pc.next = l.front
pc.prev = nil
if l.count == 0 {
l.back = pc
} else {
l.front.prev = pc
}
l.front = pc
l.count++
return
}
func (l *idleList) popFront() {
pc := l.front
l.count--
if l.count == 0 {
l.front, l.back = nil, nil
} else {
pc.next.prev = nil
l.front = pc.next
}
pc.next, pc.prev = nil, nil
}
func (l *idleList) popBack() {
pc := l.back
l.count--
if l.count == 0 {
l.front, l.back = nil, nil
} else {
pc.prev.next = nil
l.back = pc.prev
}
pc.next, pc.prev = nil, nil
}

35
src/vendor/github.com/gomodule/redigo/redis/pool17.go generated vendored Normal file
View File

@ -0,0 +1,35 @@
// Copyright 2018 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
// +build go1.7
package redis
import "context"
// GetContext gets a connection using the provided context.
//
// The provided Context must be non-nil. If the context expires before the
// connection is complete, an error is returned. Any expiration on the context
// will not affect the returned connection.
//
// If the function completes without error, then the application must close the
// returned connection.
func (p *Pool) GetContext(ctx context.Context) (Conn, error) {
pc, err := p.get(ctx)
if err != nil {
return errorConn{err}, err
}
return &activeConn{p: p, pc: pc}, nil
}

View File

@ -0,0 +1,74 @@
// Copyright 2018 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
// +build go1.7
package redis_test
import (
"context"
"testing"
"github.com/gomodule/redigo/redis"
)
func TestWaitPoolGetContext(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 1,
MaxActive: 1,
Dial: d.dial,
Wait: true,
}
defer p.Close()
c, err := p.GetContext(context.Background())
if err != nil {
t.Fatalf("GetContext returned %v", err)
}
defer c.Close()
}
func TestWaitPoolGetAfterClose(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 1,
MaxActive: 1,
Dial: d.dial,
Wait: true,
}
p.Close()
_, err := p.GetContext(context.Background())
if err == nil {
t.Fatal("expected error")
}
}
func TestWaitPoolGetCanceledContext(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 1,
MaxActive: 1,
Dial: d.dial,
Wait: true,
}
defer p.Close()
ctx, f := context.WithCancel(context.Background())
f()
c := p.Get()
defer c.Close()
_, err := p.GetContext(ctx)
if err != context.Canceled {
t.Fatalf("got error %v, want %v", err, context.Canceled)
}
}

View File

@ -0,0 +1,746 @@
// Copyright 2011 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis_test
import (
"errors"
"io"
"reflect"
"sync"
"testing"
"time"
"github.com/gomodule/redigo/redis"
)
type poolTestConn struct {
d *poolDialer
err error
redis.Conn
}
func (c *poolTestConn) Close() error {
c.d.mu.Lock()
c.d.open -= 1
c.d.mu.Unlock()
return c.Conn.Close()
}
func (c *poolTestConn) Err() error { return c.err }
func (c *poolTestConn) Do(commandName string, args ...interface{}) (interface{}, error) {
if commandName == "ERR" {
c.err = args[0].(error)
commandName = "PING"
}
if commandName != "" {
c.d.commands = append(c.d.commands, commandName)
}
return c.Conn.Do(commandName, args...)
}
func (c *poolTestConn) Send(commandName string, args ...interface{}) error {
c.d.commands = append(c.d.commands, commandName)
return c.Conn.Send(commandName, args...)
}
type poolDialer struct {
mu sync.Mutex
t *testing.T
dialed int
open int
commands []string
dialErr error
}
func (d *poolDialer) dial() (redis.Conn, error) {
d.mu.Lock()
d.dialed += 1
dialErr := d.dialErr
d.mu.Unlock()
if dialErr != nil {
return nil, d.dialErr
}
c, err := redis.DialDefaultServer()
if err != nil {
return nil, err
}
d.mu.Lock()
d.open += 1
d.mu.Unlock()
return &poolTestConn{d: d, Conn: c}, nil
}
func (d *poolDialer) check(message string, p *redis.Pool, dialed, open, inuse int) {
d.mu.Lock()
if d.dialed != dialed {
d.t.Errorf("%s: dialed=%d, want %d", message, d.dialed, dialed)
}
if d.open != open {
d.t.Errorf("%s: open=%d, want %d", message, d.open, open)
}
stats := p.Stats()
if stats.ActiveCount != open {
d.t.Errorf("%s: active=%d, want %d", message, stats.ActiveCount, open)
}
if stats.IdleCount != open-inuse {
d.t.Errorf("%s: idle=%d, want %d", message, stats.IdleCount, open-inuse)
}
d.mu.Unlock()
}
func TestPoolReuse(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 2,
Dial: d.dial,
}
for i := 0; i < 10; i++ {
c1 := p.Get()
c1.Do("PING")
c2 := p.Get()
c2.Do("PING")
c1.Close()
c2.Close()
}
d.check("before close", p, 2, 2, 0)
p.Close()
d.check("after close", p, 2, 0, 0)
}
func TestPoolMaxIdle(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 2,
Dial: d.dial,
}
defer p.Close()
for i := 0; i < 10; i++ {
c1 := p.Get()
c1.Do("PING")
c2 := p.Get()
c2.Do("PING")
c3 := p.Get()
c3.Do("PING")
c1.Close()
c2.Close()
c3.Close()
}
d.check("before close", p, 12, 2, 0)
p.Close()
d.check("after close", p, 12, 0, 0)
}
func TestPoolError(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 2,
Dial: d.dial,
}
defer p.Close()
c := p.Get()
c.Do("ERR", io.EOF)
if c.Err() == nil {
t.Errorf("expected c.Err() != nil")
}
c.Close()
c = p.Get()
c.Do("ERR", io.EOF)
c.Close()
d.check(".", p, 2, 0, 0)
}
func TestPoolClose(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 2,
Dial: d.dial,
}
defer p.Close()
c1 := p.Get()
c1.Do("PING")
c2 := p.Get()
c2.Do("PING")
c3 := p.Get()
c3.Do("PING")
c1.Close()
if _, err := c1.Do("PING"); err == nil {
t.Errorf("expected error after connection closed")
}
c2.Close()
c2.Close()
p.Close()
d.check("after pool close", p, 3, 1, 1)
if _, err := c1.Do("PING"); err == nil {
t.Errorf("expected error after connection and pool closed")
}
c3.Close()
d.check("after conn close", p, 3, 0, 0)
c1 = p.Get()
if _, err := c1.Do("PING"); err == nil {
t.Errorf("expected error after pool closed")
}
}
func TestPoolClosedConn(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 2,
IdleTimeout: 300 * time.Second,
Dial: d.dial,
}
defer p.Close()
c := p.Get()
if c.Err() != nil {
t.Fatal("get failed")
}
c.Close()
if err := c.Err(); err == nil {
t.Fatal("Err on closed connection did not return error")
}
if _, err := c.Do("PING"); err == nil {
t.Fatal("Do on closed connection did not return error")
}
if err := c.Send("PING"); err == nil {
t.Fatal("Send on closed connection did not return error")
}
if err := c.Flush(); err == nil {
t.Fatal("Flush on closed connection did not return error")
}
if _, err := c.Receive(); err == nil {
t.Fatal("Receive on closed connection did not return error")
}
}
func TestPoolIdleTimeout(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 2,
IdleTimeout: 300 * time.Second,
Dial: d.dial,
}
defer p.Close()
now := time.Now()
redis.SetNowFunc(func() time.Time { return now })
defer redis.SetNowFunc(time.Now)
c := p.Get()
c.Do("PING")
c.Close()
d.check("1", p, 1, 1, 0)
now = now.Add(p.IdleTimeout + 1)
c = p.Get()
c.Do("PING")
c.Close()
d.check("2", p, 2, 1, 0)
}
func TestPoolMaxLifetime(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 2,
MaxConnLifetime: 300 * time.Second,
Dial: d.dial,
}
defer p.Close()
now := time.Now()
redis.SetNowFunc(func() time.Time { return now })
defer redis.SetNowFunc(time.Now)
c := p.Get()
c.Do("PING")
c.Close()
d.check("1", p, 1, 1, 0)
now = now.Add(p.MaxConnLifetime + 1)
c = p.Get()
c.Do("PING")
c.Close()
d.check("2", p, 2, 1, 0)
}
func TestPoolConcurrenSendReceive(t *testing.T) {
p := &redis.Pool{
Dial: redis.DialDefaultServer,
}
defer p.Close()
c := p.Get()
done := make(chan error, 1)
go func() {
_, err := c.Receive()
done <- err
}()
c.Send("PING")
c.Flush()
err := <-done
if err != nil {
t.Fatalf("Receive() returned error %v", err)
}
_, err = c.Do("")
if err != nil {
t.Fatalf("Do() returned error %v", err)
}
c.Close()
}
func TestPoolBorrowCheck(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 2,
Dial: d.dial,
TestOnBorrow: func(redis.Conn, time.Time) error { return redis.Error("BLAH") },
}
defer p.Close()
for i := 0; i < 10; i++ {
c := p.Get()
c.Do("PING")
c.Close()
}
d.check("1", p, 10, 1, 0)
}
func TestPoolMaxActive(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 2,
MaxActive: 2,
Dial: d.dial,
}
defer p.Close()
c1 := p.Get()
c1.Do("PING")
c2 := p.Get()
c2.Do("PING")
d.check("1", p, 2, 2, 2)
c3 := p.Get()
if _, err := c3.Do("PING"); err != redis.ErrPoolExhausted {
t.Errorf("expected pool exhausted")
}
c3.Close()
d.check("2", p, 2, 2, 2)
c2.Close()
d.check("3", p, 2, 2, 1)
c3 = p.Get()
if _, err := c3.Do("PING"); err != nil {
t.Errorf("expected good channel, err=%v", err)
}
c3.Close()
d.check("4", p, 2, 2, 1)
}
func TestPoolMonitorCleanup(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 2,
MaxActive: 2,
Dial: d.dial,
}
defer p.Close()
c := p.Get()
c.Send("MONITOR")
c.Close()
d.check("", p, 1, 0, 0)
}
func TestPoolPubSubCleanup(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 2,
MaxActive: 2,
Dial: d.dial,
}
defer p.Close()
c := p.Get()
c.Send("SUBSCRIBE", "x")
c.Close()
want := []string{"SUBSCRIBE", "UNSUBSCRIBE", "PUNSUBSCRIBE", "ECHO"}
if !reflect.DeepEqual(d.commands, want) {
t.Errorf("got commands %v, want %v", d.commands, want)
}
d.commands = nil
c = p.Get()
c.Send("PSUBSCRIBE", "x*")
c.Close()
want = []string{"PSUBSCRIBE", "UNSUBSCRIBE", "PUNSUBSCRIBE", "ECHO"}
if !reflect.DeepEqual(d.commands, want) {
t.Errorf("got commands %v, want %v", d.commands, want)
}
d.commands = nil
}
func TestPoolTransactionCleanup(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 2,
MaxActive: 2,
Dial: d.dial,
}
defer p.Close()
c := p.Get()
c.Do("WATCH", "key")
c.Do("PING")
c.Close()
want := []string{"WATCH", "PING", "UNWATCH"}
if !reflect.DeepEqual(d.commands, want) {
t.Errorf("got commands %v, want %v", d.commands, want)
}
d.commands = nil
c = p.Get()
c.Do("WATCH", "key")
c.Do("UNWATCH")
c.Do("PING")
c.Close()
want = []string{"WATCH", "UNWATCH", "PING"}
if !reflect.DeepEqual(d.commands, want) {
t.Errorf("got commands %v, want %v", d.commands, want)
}
d.commands = nil
c = p.Get()
c.Do("WATCH", "key")
c.Do("MULTI")
c.Do("PING")
c.Close()
want = []string{"WATCH", "MULTI", "PING", "DISCARD"}
if !reflect.DeepEqual(d.commands, want) {
t.Errorf("got commands %v, want %v", d.commands, want)
}
d.commands = nil
c = p.Get()
c.Do("WATCH", "key")
c.Do("MULTI")
c.Do("DISCARD")
c.Do("PING")
c.Close()
want = []string{"WATCH", "MULTI", "DISCARD", "PING"}
if !reflect.DeepEqual(d.commands, want) {
t.Errorf("got commands %v, want %v", d.commands, want)
}
d.commands = nil
c = p.Get()
c.Do("WATCH", "key")
c.Do("MULTI")
c.Do("EXEC")
c.Do("PING")
c.Close()
want = []string{"WATCH", "MULTI", "EXEC", "PING"}
if !reflect.DeepEqual(d.commands, want) {
t.Errorf("got commands %v, want %v", d.commands, want)
}
d.commands = nil
}
func startGoroutines(p *redis.Pool, cmd string, args ...interface{}) chan error {
errs := make(chan error, 10)
for i := 0; i < cap(errs); i++ {
go func() {
c := p.Get()
_, err := c.Do(cmd, args...)
c.Close()
errs <- err
}()
}
return errs
}
func TestWaitPool(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 1,
MaxActive: 1,
Dial: d.dial,
Wait: true,
}
defer p.Close()
c := p.Get()
errs := startGoroutines(p, "PING")
d.check("before close", p, 1, 1, 1)
c.Close()
timeout := time.After(2 * time.Second)
for i := 0; i < cap(errs); i++ {
select {
case err := <-errs:
if err != nil {
t.Fatal(err)
}
case <-timeout:
t.Fatalf("timeout waiting for blocked goroutine %d", i)
}
}
d.check("done", p, 1, 1, 0)
}
func TestWaitPoolClose(t *testing.T) {
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 1,
MaxActive: 1,
Dial: d.dial,
Wait: true,
}
defer p.Close()
c := p.Get()
if _, err := c.Do("PING"); err != nil {
t.Fatal(err)
}
errs := startGoroutines(p, "PING")
d.check("before close", p, 1, 1, 1)
p.Close()
timeout := time.After(2 * time.Second)
for i := 0; i < cap(errs); i++ {
select {
case err := <-errs:
switch err {
case nil:
t.Fatal("blocked goroutine did not get error")
case redis.ErrPoolExhausted:
t.Fatal("blocked goroutine got pool exhausted error")
}
case <-timeout:
t.Fatal("timeout waiting for blocked goroutine")
}
}
c.Close()
d.check("done", p, 1, 0, 0)
}
func TestWaitPoolCommandError(t *testing.T) {
testErr := errors.New("test")
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 1,
MaxActive: 1,
Dial: d.dial,
Wait: true,
}
defer p.Close()
c := p.Get()
errs := startGoroutines(p, "ERR", testErr)
d.check("before close", p, 1, 1, 1)
c.Close()
timeout := time.After(2 * time.Second)
for i := 0; i < cap(errs); i++ {
select {
case err := <-errs:
if err != nil {
t.Fatal(err)
}
case <-timeout:
t.Fatalf("timeout waiting for blocked goroutine %d", i)
}
}
d.check("done", p, cap(errs), 0, 0)
}
func TestWaitPoolDialError(t *testing.T) {
testErr := errors.New("test")
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: 1,
MaxActive: 1,
Dial: d.dial,
Wait: true,
}
defer p.Close()
c := p.Get()
errs := startGoroutines(p, "ERR", testErr)
d.check("before close", p, 1, 1, 1)
d.dialErr = errors.New("dial")
c.Close()
nilCount := 0
errCount := 0
timeout := time.After(2 * time.Second)
for i := 0; i < cap(errs); i++ {
select {
case err := <-errs:
switch err {
case nil:
nilCount++
case d.dialErr:
errCount++
default:
t.Fatalf("expected dial error or nil, got %v", err)
}
case <-timeout:
t.Fatalf("timeout waiting for blocked goroutine %d", i)
}
}
if nilCount != 1 {
t.Errorf("expected one nil error, got %d", nilCount)
}
if errCount != cap(errs)-1 {
t.Errorf("expected %d dial errors, got %d", cap(errs)-1, errCount)
}
d.check("done", p, cap(errs), 0, 0)
}
// Borrowing requires us to iterate over the idle connections, unlock the pool,
// and perform a blocking operation to check the connection still works. If
// TestOnBorrow fails, we must reacquire the lock and continue iteration. This
// test ensures that iteration will work correctly if multiple threads are
// iterating simultaneously.
func TestLocking_TestOnBorrowFails_PoolDoesntCrash(t *testing.T) {
const count = 100
// First we'll Create a pool where the pilfering of idle connections fails.
d := poolDialer{t: t}
p := &redis.Pool{
MaxIdle: count,
MaxActive: count,
Dial: d.dial,
TestOnBorrow: func(c redis.Conn, t time.Time) error {
return errors.New("No way back into the real world.")
},
}
defer p.Close()
// Fill the pool with idle connections.
conns := make([]redis.Conn, count)
for i := range conns {
conns[i] = p.Get()
}
for i := range conns {
conns[i].Close()
}
// Spawn a bunch of goroutines to thrash the pool.
var wg sync.WaitGroup
wg.Add(count)
for i := 0; i < count; i++ {
go func() {
c := p.Get()
if c.Err() != nil {
t.Errorf("pool get failed: %v", c.Err())
}
c.Close()
wg.Done()
}()
}
wg.Wait()
if d.dialed != count*2 {
t.Errorf("Expected %d dials, got %d", count*2, d.dialed)
}
}
func BenchmarkPoolGet(b *testing.B) {
b.StopTimer()
p := redis.Pool{Dial: redis.DialDefaultServer, MaxIdle: 2}
c := p.Get()
if err := c.Err(); err != nil {
b.Fatal(err)
}
c.Close()
defer p.Close()
b.StartTimer()
for i := 0; i < b.N; i++ {
c = p.Get()
c.Close()
}
}
func BenchmarkPoolGetErr(b *testing.B) {
b.StopTimer()
p := redis.Pool{Dial: redis.DialDefaultServer, MaxIdle: 2}
c := p.Get()
if err := c.Err(); err != nil {
b.Fatal(err)
}
c.Close()
defer p.Close()
b.StartTimer()
for i := 0; i < b.N; i++ {
c = p.Get()
if err := c.Err(); err != nil {
b.Fatal(err)
}
c.Close()
}
}
func BenchmarkPoolGetPing(b *testing.B) {
b.StopTimer()
p := redis.Pool{Dial: redis.DialDefaultServer, MaxIdle: 2}
c := p.Get()
if err := c.Err(); err != nil {
b.Fatal(err)
}
c.Close()
defer p.Close()
b.StartTimer()
for i := 0; i < b.N; i++ {
c = p.Get()
if _, err := c.Do("PING"); err != nil {
b.Fatal(err)
}
c.Close()
}
}

148
src/vendor/github.com/gomodule/redigo/redis/pubsub.go generated vendored Normal file
View File

@ -0,0 +1,148 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis
import (
"errors"
"time"
)
// Subscription represents a subscribe or unsubscribe notification.
type Subscription struct {
// Kind is "subscribe", "unsubscribe", "psubscribe" or "punsubscribe"
Kind string
// The channel that was changed.
Channel string
// The current number of subscriptions for connection.
Count int
}
// Message represents a message notification.
type Message struct {
// The originating channel.
Channel string
// The matched pattern, if any
Pattern string
// The message data.
Data []byte
}
// Pong represents a pubsub pong notification.
type Pong struct {
Data string
}
// PubSubConn wraps a Conn with convenience methods for subscribers.
type PubSubConn struct {
Conn Conn
}
// Close closes the connection.
func (c PubSubConn) Close() error {
return c.Conn.Close()
}
// Subscribe subscribes the connection to the specified channels.
func (c PubSubConn) Subscribe(channel ...interface{}) error {
c.Conn.Send("SUBSCRIBE", channel...)
return c.Conn.Flush()
}
// PSubscribe subscribes the connection to the given patterns.
func (c PubSubConn) PSubscribe(channel ...interface{}) error {
c.Conn.Send("PSUBSCRIBE", channel...)
return c.Conn.Flush()
}
// Unsubscribe unsubscribes the connection from the given channels, or from all
// of them if none is given.
func (c PubSubConn) Unsubscribe(channel ...interface{}) error {
c.Conn.Send("UNSUBSCRIBE", channel...)
return c.Conn.Flush()
}
// PUnsubscribe unsubscribes the connection from the given patterns, or from all
// of them if none is given.
func (c PubSubConn) PUnsubscribe(channel ...interface{}) error {
c.Conn.Send("PUNSUBSCRIBE", channel...)
return c.Conn.Flush()
}
// Ping sends a PING to the server with the specified data.
//
// The connection must be subscribed to at least one channel or pattern when
// calling this method.
func (c PubSubConn) Ping(data string) error {
c.Conn.Send("PING", data)
return c.Conn.Flush()
}
// Receive returns a pushed message as a Subscription, Message, Pong or error.
// The return value is intended to be used directly in a type switch as
// illustrated in the PubSubConn example.
func (c PubSubConn) Receive() interface{} {
return c.receiveInternal(c.Conn.Receive())
}
// ReceiveWithTimeout is like Receive, but it allows the application to
// override the connection's default timeout.
func (c PubSubConn) ReceiveWithTimeout(timeout time.Duration) interface{} {
return c.receiveInternal(ReceiveWithTimeout(c.Conn, timeout))
}
func (c PubSubConn) receiveInternal(replyArg interface{}, errArg error) interface{} {
reply, err := Values(replyArg, errArg)
if err != nil {
return err
}
var kind string
reply, err = Scan(reply, &kind)
if err != nil {
return err
}
switch kind {
case "message":
var m Message
if _, err := Scan(reply, &m.Channel, &m.Data); err != nil {
return err
}
return m
case "pmessage":
var m Message
if _, err := Scan(reply, &m.Pattern, &m.Channel, &m.Data); err != nil {
return err
}
return m
case "subscribe", "psubscribe", "unsubscribe", "punsubscribe":
s := Subscription{Kind: kind}
if _, err := Scan(reply, &s.Channel, &s.Count); err != nil {
return err
}
return s
case "pong":
var p Pong
if _, err := Scan(reply, &p.Data); err != nil {
return err
}
return p
}
return errors.New("redigo: unknown pubsub notification")
}

View File

@ -0,0 +1,165 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
// +build go1.7
package redis_test
import (
"context"
"fmt"
"time"
"github.com/gomodule/redigo/redis"
)
// listenPubSubChannels listens for messages on Redis pubsub channels. The
// onStart function is called after the channels are subscribed. The onMessage
// function is called for each message.
func listenPubSubChannels(ctx context.Context, redisServerAddr string,
onStart func() error,
onMessage func(channel string, data []byte) error,
channels ...string) error {
// A ping is set to the server with this period to test for the health of
// the connection and server.
const healthCheckPeriod = time.Minute
c, err := redis.Dial("tcp", redisServerAddr,
// Read timeout on server should be greater than ping period.
redis.DialReadTimeout(healthCheckPeriod+10*time.Second),
redis.DialWriteTimeout(10*time.Second))
if err != nil {
return err
}
defer c.Close()
psc := redis.PubSubConn{Conn: c}
if err := psc.Subscribe(redis.Args{}.AddFlat(channels)...); err != nil {
return err
}
done := make(chan error, 1)
// Start a goroutine to receive notifications from the server.
go func() {
for {
switch n := psc.Receive().(type) {
case error:
done <- n
return
case redis.Message:
if err := onMessage(n.Channel, n.Data); err != nil {
done <- err
return
}
case redis.Subscription:
switch n.Count {
case len(channels):
// Notify application when all channels are subscribed.
if err := onStart(); err != nil {
done <- err
return
}
case 0:
// Return from the goroutine when all channels are unsubscribed.
done <- nil
return
}
}
}
}()
ticker := time.NewTicker(healthCheckPeriod)
defer ticker.Stop()
loop:
for err == nil {
select {
case <-ticker.C:
// Send ping to test health of connection and server. If
// corresponding pong is not received, then receive on the
// connection will timeout and the receive goroutine will exit.
if err = psc.Ping(""); err != nil {
break loop
}
case <-ctx.Done():
break loop
case err := <-done:
// Return error from the receive goroutine.
return err
}
}
// Signal the receiving goroutine to exit by unsubscribing from all channels.
psc.Unsubscribe()
// Wait for goroutine to complete.
return <-done
}
func publish() {
c, err := dial()
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
c.Do("PUBLISH", "c1", "hello")
c.Do("PUBLISH", "c2", "world")
c.Do("PUBLISH", "c1", "goodbye")
}
// This example shows how receive pubsub notifications with cancelation and
// health checks.
func ExamplePubSubConn() {
redisServerAddr, err := serverAddr()
if err != nil {
fmt.Println(err)
return
}
ctx, cancel := context.WithCancel(context.Background())
err = listenPubSubChannels(ctx,
redisServerAddr,
func() error {
// The start callback is a good place to backfill missed
// notifications. For the purpose of this example, a goroutine is
// started to send notifications.
go publish()
return nil
},
func(channel string, message []byte) error {
fmt.Printf("channel: %s, message: %s\n", channel, message)
// For the purpose of this example, cancel the listener's context
// after receiving last message sent by publish().
if string(message) == "goodbye" {
cancel()
}
return nil
},
"c1", "c2")
if err != nil {
fmt.Println(err)
return
}
// Output:
// channel: c1, message: hello
// channel: c2, message: world
// channel: c1, message: goodbye
}

View File

@ -0,0 +1,74 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis_test
import (
"reflect"
"testing"
"time"
"github.com/gomodule/redigo/redis"
)
func expectPushed(t *testing.T, c redis.PubSubConn, message string, expected interface{}) {
actual := c.Receive()
if !reflect.DeepEqual(actual, expected) {
t.Errorf("%s = %v, want %v", message, actual, expected)
}
}
func TestPushed(t *testing.T) {
pc, err := redis.DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer pc.Close()
sc, err := redis.DialDefaultServer()
if err != nil {
t.Fatalf("error connection to database, %v", err)
}
defer sc.Close()
c := redis.PubSubConn{Conn: sc}
c.Subscribe("c1")
expectPushed(t, c, "Subscribe(c1)", redis.Subscription{Kind: "subscribe", Channel: "c1", Count: 1})
c.Subscribe("c2")
expectPushed(t, c, "Subscribe(c2)", redis.Subscription{Kind: "subscribe", Channel: "c2", Count: 2})
c.PSubscribe("p1")
expectPushed(t, c, "PSubscribe(p1)", redis.Subscription{Kind: "psubscribe", Channel: "p1", Count: 3})
c.PSubscribe("p2")
expectPushed(t, c, "PSubscribe(p2)", redis.Subscription{Kind: "psubscribe", Channel: "p2", Count: 4})
c.PUnsubscribe()
expectPushed(t, c, "Punsubscribe(p1)", redis.Subscription{Kind: "punsubscribe", Channel: "p1", Count: 3})
expectPushed(t, c, "Punsubscribe()", redis.Subscription{Kind: "punsubscribe", Channel: "p2", Count: 2})
pc.Do("PUBLISH", "c1", "hello")
expectPushed(t, c, "PUBLISH c1 hello", redis.Message{Channel: "c1", Data: []byte("hello")})
c.Ping("hello")
expectPushed(t, c, `Ping("hello")`, redis.Pong{Data: "hello"})
c.Conn.Send("PING")
c.Conn.Flush()
expectPushed(t, c, `Send("PING")`, redis.Pong{})
c.Ping("timeout")
got := c.ReceiveWithTimeout(time.Minute)
if want := (redis.Pong{Data: "timeout"}); want != got {
t.Errorf("recv /w timeout got %v, want %v", got, want)
}
}

117
src/vendor/github.com/gomodule/redigo/redis/redis.go generated vendored Normal file
View File

@ -0,0 +1,117 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis
import (
"errors"
"time"
)
// Error represents an error returned in a command reply.
type Error string
func (err Error) Error() string { return string(err) }
// Conn represents a connection to a Redis server.
type Conn interface {
// Close closes the connection.
Close() error
// Err returns a non-nil value when the connection is not usable.
Err() error
// Do sends a command to the server and returns the received reply.
Do(commandName string, args ...interface{}) (reply interface{}, err error)
// Send writes the command to the client's output buffer.
Send(commandName string, args ...interface{}) error
// Flush flushes the output buffer to the Redis server.
Flush() error
// Receive receives a single reply from the Redis server
Receive() (reply interface{}, err error)
}
// Argument is the interface implemented by an object which wants to control how
// the object is converted to Redis bulk strings.
type Argument interface {
// RedisArg returns a value to be encoded as a bulk string per the
// conversions listed in the section 'Executing Commands'.
// Implementations should typically return a []byte or string.
RedisArg() interface{}
}
// Scanner is implemented by an object which wants to control its value is
// interpreted when read from Redis.
type Scanner interface {
// RedisScan assigns a value from a Redis value. The argument src is one of
// the reply types listed in the section `Executing Commands`.
//
// An error should be returned if the value cannot be stored without
// loss of information.
RedisScan(src interface{}) error
}
// ConnWithTimeout is an optional interface that allows the caller to override
// a connection's default read timeout. This interface is useful for executing
// the BLPOP, BRPOP, BRPOPLPUSH, XREAD and other commands that block at the
// server.
//
// A connection's default read timeout is set with the DialReadTimeout dial
// option. Applications should rely on the default timeout for commands that do
// not block at the server.
//
// All of the Conn implementations in this package satisfy the ConnWithTimeout
// interface.
//
// Use the DoWithTimeout and ReceiveWithTimeout helper functions to simplify
// use of this interface.
type ConnWithTimeout interface {
Conn
// Do sends a command to the server and returns the received reply.
// The timeout overrides the read timeout set when dialing the
// connection.
DoWithTimeout(timeout time.Duration, commandName string, args ...interface{}) (reply interface{}, err error)
// Receive receives a single reply from the Redis server. The timeout
// overrides the read timeout set when dialing the connection.
ReceiveWithTimeout(timeout time.Duration) (reply interface{}, err error)
}
var errTimeoutNotSupported = errors.New("redis: connection does not support ConnWithTimeout")
// DoWithTimeout executes a Redis command with the specified read timeout. If
// the connection does not satisfy the ConnWithTimeout interface, then an error
// is returned.
func DoWithTimeout(c Conn, timeout time.Duration, cmd string, args ...interface{}) (interface{}, error) {
cwt, ok := c.(ConnWithTimeout)
if !ok {
return nil, errTimeoutNotSupported
}
return cwt.DoWithTimeout(timeout, cmd, args...)
}
// ReceiveWithTimeout receives a reply with the specified read timeout. If the
// connection does not satisfy the ConnWithTimeout interface, then an error is
// returned.
func ReceiveWithTimeout(c Conn, timeout time.Duration) (interface{}, error) {
cwt, ok := c.(ConnWithTimeout)
if !ok {
return nil, errTimeoutNotSupported
}
return cwt.ReceiveWithTimeout(timeout)
}

View File

@ -0,0 +1,71 @@
// Copyright 2017 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis_test
import (
"testing"
"time"
"github.com/gomodule/redigo/redis"
)
type timeoutTestConn int
func (tc timeoutTestConn) Do(string, ...interface{}) (interface{}, error) {
return time.Duration(-1), nil
}
func (tc timeoutTestConn) DoWithTimeout(timeout time.Duration, cmd string, args ...interface{}) (interface{}, error) {
return timeout, nil
}
func (tc timeoutTestConn) Receive() (interface{}, error) {
return time.Duration(-1), nil
}
func (tc timeoutTestConn) ReceiveWithTimeout(timeout time.Duration) (interface{}, error) {
return timeout, nil
}
func (tc timeoutTestConn) Send(string, ...interface{}) error { return nil }
func (tc timeoutTestConn) Err() error { return nil }
func (tc timeoutTestConn) Close() error { return nil }
func (tc timeoutTestConn) Flush() error { return nil }
func testTimeout(t *testing.T, c redis.Conn) {
r, err := c.Do("PING")
if r != time.Duration(-1) || err != nil {
t.Errorf("Do() = %v, %v, want %v, %v", r, err, time.Duration(-1), nil)
}
r, err = redis.DoWithTimeout(c, time.Minute, "PING")
if r != time.Minute || err != nil {
t.Errorf("DoWithTimeout() = %v, %v, want %v, %v", r, err, time.Minute, nil)
}
r, err = c.Receive()
if r != time.Duration(-1) || err != nil {
t.Errorf("Receive() = %v, %v, want %v, %v", r, err, time.Duration(-1), nil)
}
r, err = redis.ReceiveWithTimeout(c, time.Minute)
if r != time.Minute || err != nil {
t.Errorf("ReceiveWithTimeout() = %v, %v, want %v, %v", r, err, time.Minute, nil)
}
}
func TestConnTimeout(t *testing.T) {
testTimeout(t, timeoutTestConn(0))
}
func TestPoolConnTimeout(t *testing.T) {
p := &redis.Pool{Dial: func() (redis.Conn, error) { return timeoutTestConn(0), nil }}
testTimeout(t, p.Get())
}

479
src/vendor/github.com/gomodule/redigo/redis/reply.go generated vendored Normal file
View File

@ -0,0 +1,479 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis
import (
"errors"
"fmt"
"strconv"
)
// ErrNil indicates that a reply value is nil.
var ErrNil = errors.New("redigo: nil returned")
// Int is a helper that converts a command reply to an integer. If err is not
// equal to nil, then Int returns 0, err. Otherwise, Int converts the
// reply to an int as follows:
//
// Reply type Result
// integer int(reply), nil
// bulk string parsed reply, nil
// nil 0, ErrNil
// other 0, error
func Int(reply interface{}, err error) (int, error) {
if err != nil {
return 0, err
}
switch reply := reply.(type) {
case int64:
x := int(reply)
if int64(x) != reply {
return 0, strconv.ErrRange
}
return x, nil
case []byte:
n, err := strconv.ParseInt(string(reply), 10, 0)
return int(n), err
case nil:
return 0, ErrNil
case Error:
return 0, reply
}
return 0, fmt.Errorf("redigo: unexpected type for Int, got type %T", reply)
}
// Int64 is a helper that converts a command reply to 64 bit integer. If err is
// not equal to nil, then Int returns 0, err. Otherwise, Int64 converts the
// reply to an int64 as follows:
//
// Reply type Result
// integer reply, nil
// bulk string parsed reply, nil
// nil 0, ErrNil
// other 0, error
func Int64(reply interface{}, err error) (int64, error) {
if err != nil {
return 0, err
}
switch reply := reply.(type) {
case int64:
return reply, nil
case []byte:
n, err := strconv.ParseInt(string(reply), 10, 64)
return n, err
case nil:
return 0, ErrNil
case Error:
return 0, reply
}
return 0, fmt.Errorf("redigo: unexpected type for Int64, got type %T", reply)
}
var errNegativeInt = errors.New("redigo: unexpected value for Uint64")
// Uint64 is a helper that converts a command reply to 64 bit integer. If err is
// not equal to nil, then Int returns 0, err. Otherwise, Int64 converts the
// reply to an int64 as follows:
//
// Reply type Result
// integer reply, nil
// bulk string parsed reply, nil
// nil 0, ErrNil
// other 0, error
func Uint64(reply interface{}, err error) (uint64, error) {
if err != nil {
return 0, err
}
switch reply := reply.(type) {
case int64:
if reply < 0 {
return 0, errNegativeInt
}
return uint64(reply), nil
case []byte:
n, err := strconv.ParseUint(string(reply), 10, 64)
return n, err
case nil:
return 0, ErrNil
case Error:
return 0, reply
}
return 0, fmt.Errorf("redigo: unexpected type for Uint64, got type %T", reply)
}
// Float64 is a helper that converts a command reply to 64 bit float. If err is
// not equal to nil, then Float64 returns 0, err. Otherwise, Float64 converts
// the reply to an int as follows:
//
// Reply type Result
// bulk string parsed reply, nil
// nil 0, ErrNil
// other 0, error
func Float64(reply interface{}, err error) (float64, error) {
if err != nil {
return 0, err
}
switch reply := reply.(type) {
case []byte:
n, err := strconv.ParseFloat(string(reply), 64)
return n, err
case nil:
return 0, ErrNil
case Error:
return 0, reply
}
return 0, fmt.Errorf("redigo: unexpected type for Float64, got type %T", reply)
}
// String is a helper that converts a command reply to a string. If err is not
// equal to nil, then String returns "", err. Otherwise String converts the
// reply to a string as follows:
//
// Reply type Result
// bulk string string(reply), nil
// simple string reply, nil
// nil "", ErrNil
// other "", error
func String(reply interface{}, err error) (string, error) {
if err != nil {
return "", err
}
switch reply := reply.(type) {
case []byte:
return string(reply), nil
case string:
return reply, nil
case nil:
return "", ErrNil
case Error:
return "", reply
}
return "", fmt.Errorf("redigo: unexpected type for String, got type %T", reply)
}
// Bytes is a helper that converts a command reply to a slice of bytes. If err
// is not equal to nil, then Bytes returns nil, err. Otherwise Bytes converts
// the reply to a slice of bytes as follows:
//
// Reply type Result
// bulk string reply, nil
// simple string []byte(reply), nil
// nil nil, ErrNil
// other nil, error
func Bytes(reply interface{}, err error) ([]byte, error) {
if err != nil {
return nil, err
}
switch reply := reply.(type) {
case []byte:
return reply, nil
case string:
return []byte(reply), nil
case nil:
return nil, ErrNil
case Error:
return nil, reply
}
return nil, fmt.Errorf("redigo: unexpected type for Bytes, got type %T", reply)
}
// Bool is a helper that converts a command reply to a boolean. If err is not
// equal to nil, then Bool returns false, err. Otherwise Bool converts the
// reply to boolean as follows:
//
// Reply type Result
// integer value != 0, nil
// bulk string strconv.ParseBool(reply)
// nil false, ErrNil
// other false, error
func Bool(reply interface{}, err error) (bool, error) {
if err != nil {
return false, err
}
switch reply := reply.(type) {
case int64:
return reply != 0, nil
case []byte:
return strconv.ParseBool(string(reply))
case nil:
return false, ErrNil
case Error:
return false, reply
}
return false, fmt.Errorf("redigo: unexpected type for Bool, got type %T", reply)
}
// MultiBulk is a helper that converts an array command reply to a []interface{}.
//
// Deprecated: Use Values instead.
func MultiBulk(reply interface{}, err error) ([]interface{}, error) { return Values(reply, err) }
// Values is a helper that converts an array command reply to a []interface{}.
// If err is not equal to nil, then Values returns nil, err. Otherwise, Values
// converts the reply as follows:
//
// Reply type Result
// array reply, nil
// nil nil, ErrNil
// other nil, error
func Values(reply interface{}, err error) ([]interface{}, error) {
if err != nil {
return nil, err
}
switch reply := reply.(type) {
case []interface{}:
return reply, nil
case nil:
return nil, ErrNil
case Error:
return nil, reply
}
return nil, fmt.Errorf("redigo: unexpected type for Values, got type %T", reply)
}
func sliceHelper(reply interface{}, err error, name string, makeSlice func(int), assign func(int, interface{}) error) error {
if err != nil {
return err
}
switch reply := reply.(type) {
case []interface{}:
makeSlice(len(reply))
for i := range reply {
if reply[i] == nil {
continue
}
if err := assign(i, reply[i]); err != nil {
return err
}
}
return nil
case nil:
return ErrNil
case Error:
return reply
}
return fmt.Errorf("redigo: unexpected type for %s, got type %T", name, reply)
}
// Float64s is a helper that converts an array command reply to a []float64. If
// err is not equal to nil, then Float64s returns nil, err. Nil array items are
// converted to 0 in the output slice. Floats64 returns an error if an array
// item is not a bulk string or nil.
func Float64s(reply interface{}, err error) ([]float64, error) {
var result []float64
err = sliceHelper(reply, err, "Float64s", func(n int) { result = make([]float64, n) }, func(i int, v interface{}) error {
p, ok := v.([]byte)
if !ok {
return fmt.Errorf("redigo: unexpected element type for Floats64, got type %T", v)
}
f, err := strconv.ParseFloat(string(p), 64)
result[i] = f
return err
})
return result, err
}
// Strings is a helper that converts an array command reply to a []string. If
// err is not equal to nil, then Strings returns nil, err. Nil array items are
// converted to "" in the output slice. Strings returns an error if an array
// item is not a bulk string or nil.
func Strings(reply interface{}, err error) ([]string, error) {
var result []string
err = sliceHelper(reply, err, "Strings", func(n int) { result = make([]string, n) }, func(i int, v interface{}) error {
switch v := v.(type) {
case string:
result[i] = v
return nil
case []byte:
result[i] = string(v)
return nil
default:
return fmt.Errorf("redigo: unexpected element type for Strings, got type %T", v)
}
})
return result, err
}
// ByteSlices is a helper that converts an array command reply to a [][]byte.
// If err is not equal to nil, then ByteSlices returns nil, err. Nil array
// items are stay nil. ByteSlices returns an error if an array item is not a
// bulk string or nil.
func ByteSlices(reply interface{}, err error) ([][]byte, error) {
var result [][]byte
err = sliceHelper(reply, err, "ByteSlices", func(n int) { result = make([][]byte, n) }, func(i int, v interface{}) error {
p, ok := v.([]byte)
if !ok {
return fmt.Errorf("redigo: unexpected element type for ByteSlices, got type %T", v)
}
result[i] = p
return nil
})
return result, err
}
// Int64s is a helper that converts an array command reply to a []int64.
// If err is not equal to nil, then Int64s returns nil, err. Nil array
// items are stay nil. Int64s returns an error if an array item is not a
// bulk string or nil.
func Int64s(reply interface{}, err error) ([]int64, error) {
var result []int64
err = sliceHelper(reply, err, "Int64s", func(n int) { result = make([]int64, n) }, func(i int, v interface{}) error {
switch v := v.(type) {
case int64:
result[i] = v
return nil
case []byte:
n, err := strconv.ParseInt(string(v), 10, 64)
result[i] = n
return err
default:
return fmt.Errorf("redigo: unexpected element type for Int64s, got type %T", v)
}
})
return result, err
}
// Ints is a helper that converts an array command reply to a []in.
// If err is not equal to nil, then Ints returns nil, err. Nil array
// items are stay nil. Ints returns an error if an array item is not a
// bulk string or nil.
func Ints(reply interface{}, err error) ([]int, error) {
var result []int
err = sliceHelper(reply, err, "Ints", func(n int) { result = make([]int, n) }, func(i int, v interface{}) error {
switch v := v.(type) {
case int64:
n := int(v)
if int64(n) != v {
return strconv.ErrRange
}
result[i] = n
return nil
case []byte:
n, err := strconv.Atoi(string(v))
result[i] = n
return err
default:
return fmt.Errorf("redigo: unexpected element type for Ints, got type %T", v)
}
})
return result, err
}
// StringMap is a helper that converts an array of strings (alternating key, value)
// into a map[string]string. The HGETALL and CONFIG GET commands return replies in this format.
// Requires an even number of values in result.
func StringMap(result interface{}, err error) (map[string]string, error) {
values, err := Values(result, err)
if err != nil {
return nil, err
}
if len(values)%2 != 0 {
return nil, errors.New("redigo: StringMap expects even number of values result")
}
m := make(map[string]string, len(values)/2)
for i := 0; i < len(values); i += 2 {
key, okKey := values[i].([]byte)
value, okValue := values[i+1].([]byte)
if !okKey || !okValue {
return nil, errors.New("redigo: StringMap key not a bulk string value")
}
m[string(key)] = string(value)
}
return m, nil
}
// IntMap is a helper that converts an array of strings (alternating key, value)
// into a map[string]int. The HGETALL commands return replies in this format.
// Requires an even number of values in result.
func IntMap(result interface{}, err error) (map[string]int, error) {
values, err := Values(result, err)
if err != nil {
return nil, err
}
if len(values)%2 != 0 {
return nil, errors.New("redigo: IntMap expects even number of values result")
}
m := make(map[string]int, len(values)/2)
for i := 0; i < len(values); i += 2 {
key, ok := values[i].([]byte)
if !ok {
return nil, errors.New("redigo: IntMap key not a bulk string value")
}
value, err := Int(values[i+1], nil)
if err != nil {
return nil, err
}
m[string(key)] = value
}
return m, nil
}
// Int64Map is a helper that converts an array of strings (alternating key, value)
// into a map[string]int64. The HGETALL commands return replies in this format.
// Requires an even number of values in result.
func Int64Map(result interface{}, err error) (map[string]int64, error) {
values, err := Values(result, err)
if err != nil {
return nil, err
}
if len(values)%2 != 0 {
return nil, errors.New("redigo: Int64Map expects even number of values result")
}
m := make(map[string]int64, len(values)/2)
for i := 0; i < len(values); i += 2 {
key, ok := values[i].([]byte)
if !ok {
return nil, errors.New("redigo: Int64Map key not a bulk string value")
}
value, err := Int64(values[i+1], nil)
if err != nil {
return nil, err
}
m[string(key)] = value
}
return m, nil
}
// Positions is a helper that converts an array of positions (lat, long)
// into a [][2]float64. The GEOPOS command returns replies in this format.
func Positions(result interface{}, err error) ([]*[2]float64, error) {
values, err := Values(result, err)
if err != nil {
return nil, err
}
positions := make([]*[2]float64, len(values))
for i := range values {
if values[i] == nil {
continue
}
p, ok := values[i].([]interface{})
if !ok {
return nil, fmt.Errorf("redigo: unexpected element type for interface slice, got type %T", values[i])
}
if len(p) != 2 {
return nil, fmt.Errorf("redigo: unexpected number of values for a member position, got %d", len(p))
}
lat, err := Float64(p[0], nil)
if err != nil {
return nil, err
}
long, err := Float64(p[1], nil)
if err != nil {
return nil, err
}
positions[i] = &[2]float64{lat, long}
}
return positions, nil
}

View File

@ -0,0 +1,209 @@
// Copyright 2012 Gary Burd
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package redis_test
import (
"fmt"
"reflect"
"testing"
"github.com/gomodule/redigo/redis"
)
type valueError struct {
v interface{}
err error
}
func ve(v interface{}, err error) valueError {
return valueError{v, err}
}
var replyTests = []struct {
name interface{}
actual valueError
expected valueError
}{
{
"ints([[]byte, []byte])",
ve(redis.Ints([]interface{}{[]byte("4"), []byte("5")}, nil)),
ve([]int{4, 5}, nil),
},
{
"ints([nt64, int64])",
ve(redis.Ints([]interface{}{int64(4), int64(5)}, nil)),
ve([]int{4, 5}, nil),
},
{
"ints([[]byte, nil, []byte])",
ve(redis.Ints([]interface{}{[]byte("4"), nil, []byte("5")}, nil)),
ve([]int{4, 0, 5}, nil),
},
{
"ints(nil)",
ve(redis.Ints(nil, nil)),
ve([]int(nil), redis.ErrNil),
},
{
"int64s([[]byte, []byte])",
ve(redis.Int64s([]interface{}{[]byte("4"), []byte("5")}, nil)),
ve([]int64{4, 5}, nil),
},
{
"int64s([int64, int64])",
ve(redis.Int64s([]interface{}{int64(4), int64(5)}, nil)),
ve([]int64{4, 5}, nil),
},
{
"strings([[]byte, []bytev2])",
ve(redis.Strings([]interface{}{[]byte("v1"), []byte("v2")}, nil)),
ve([]string{"v1", "v2"}, nil),
},
{
"strings([string, string])",
ve(redis.Strings([]interface{}{"v1", "v2"}, nil)),
ve([]string{"v1", "v2"}, nil),
},
{
"byteslices([v1, v2])",
ve(redis.ByteSlices([]interface{}{[]byte("v1"), []byte("v2")}, nil)),
ve([][]byte{[]byte("v1"), []byte("v2")}, nil),
},
{
"float64s([v1, v2])",
ve(redis.Float64s([]interface{}{[]byte("1.234"), []byte("5.678")}, nil)),
ve([]float64{1.234, 5.678}, nil),
},
{
"values([v1, v2])",
ve(redis.Values([]interface{}{[]byte("v1"), []byte("v2")}, nil)),
ve([]interface{}{[]byte("v1"), []byte("v2")}, nil),
},
{
"values(nil)",
ve(redis.Values(nil, nil)),
ve([]interface{}(nil), redis.ErrNil),
},
{
"float64(1.0)",
ve(redis.Float64([]byte("1.0"), nil)),
ve(float64(1.0), nil),
},
{
"float64(nil)",
ve(redis.Float64(nil, nil)),
ve(float64(0.0), redis.ErrNil),
},
{
"uint64(1)",
ve(redis.Uint64(int64(1), nil)),
ve(uint64(1), nil),
},
{
"uint64(-1)",
ve(redis.Uint64(int64(-1), nil)),
ve(uint64(0), redis.ErrNegativeInt),
},
{
"positions([[1, 2], nil, [3, 4]])",
ve(redis.Positions([]interface{}{[]interface{}{[]byte("1"), []byte("2")}, nil, []interface{}{[]byte("3"), []byte("4")}}, nil)),
ve([]*[2]float64{{1.0, 2.0}, nil, {3.0, 4.0}}, nil),
},
}
func TestReply(t *testing.T) {
for _, rt := range replyTests {
if rt.actual.err != rt.expected.err {
t.Errorf("%s returned err %v, want %v", rt.name, rt.actual.err, rt.expected.err)
continue
}
if !reflect.DeepEqual(rt.actual.v, rt.expected.v) {
t.Errorf("%s=%+v, want %+v", rt.name, rt.actual.v, rt.expected.v)
}
}
}
// dial wraps DialDefaultServer() with a more suitable function name for examples.
func dial() (redis.Conn, error) {
return redis.DialDefaultServer()
}
// serverAddr wraps DefaultServerAddr() with a more suitable function name for examples.
func serverAddr() (string, error) {
return redis.DefaultServerAddr()
}
func ExampleBool() {
c, err := dial()
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
c.Do("SET", "foo", 1)
exists, _ := redis.Bool(c.Do("EXISTS", "foo"))
fmt.Printf("%#v\n", exists)
// Output:
// true
}
func ExampleInt() {
c, err := dial()
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
c.Do("SET", "k1", 1)
n, _ := redis.Int(c.Do("GET", "k1"))
fmt.Printf("%#v\n", n)
n, _ = redis.Int(c.Do("INCR", "k1"))
fmt.Printf("%#v\n", n)
// Output:
// 1
// 2
}
func ExampleInts() {
c, err := dial()
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
c.Do("SADD", "set_with_integers", 4, 5, 6)
ints, _ := redis.Ints(c.Do("SMEMBERS", "set_with_integers"))
fmt.Printf("%#v\n", ints)
// Output:
// []int{4, 5, 6}
}
func ExampleString() {
c, err := dial()
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
c.Do("SET", "hello", "world")
s, err := redis.String(c.Do("GET", "hello"))
fmt.Printf("%#v\n", s)
// Output:
// "world"
}

Some files were not shown because too many files have changed in this diff Show More