async/multi_pop uuids
Brightcells
5 changed files with 61 additions and 9 deletions
+ 14
- 0
pai2/settings.py
@@ -44,6 +44,7 @@ INSTALLED_APPS = ( |
||
| 44 | 44 |
'django.contrib.messages', |
| 45 | 45 |
'django.contrib.staticfiles', |
| 46 | 46 |
'rest_framework', |
| 47 |
+ 'django_q', |
|
| 47 | 48 |
'api', |
| 48 | 49 |
'account', |
| 49 | 50 |
'group', |
@@ -352,5 +353,18 @@ except ImportError: |
||
| 352 | 353 |
try: |
| 353 | 354 |
from func_settings import redis_connect |
| 354 | 355 |
REDIS_CACHE = redis_connect(REDIS.get('default', {}))
|
| 356 |
+ |
|
| 357 |
+ Q_CLUSTER = {
|
|
| 358 |
+ 'name': 'pai2', |
|
| 359 |
+ 'workers': 8, |
|
| 360 |
+ 'recycle': 500, |
|
| 361 |
+ 'timeout': 60, |
|
| 362 |
+ 'compress': True, |
|
| 363 |
+ 'cpu_affinity': 1, |
|
| 364 |
+ 'save_limit': 250, |
|
| 365 |
+ 'queue_limit': 500, |
|
| 366 |
+ 'label': 'Django Q', |
|
| 367 |
+ 'redis_conn': REDIS_CACHE, |
|
| 368 |
+ } |
|
| 355 | 369 |
except ImportError: |
| 356 | 370 |
REDIS_CACHE = None |
+ 17
- 9
photo/views.py
@@ -8,7 +8,9 @@ from django.conf import settings |
||
| 8 | 8 |
from django.core.files.storage import default_storage |
| 9 | 9 |
from django.db import transaction |
| 10 | 10 |
from django.shortcuts import render |
| 11 |
+from django_q.tasks import async |
|
| 11 | 12 |
from logit import logit |
| 13 |
+from redis_extensions import multi_pop |
|
| 12 | 14 |
from rest_framework import viewsets |
| 13 | 15 |
from TimeConvert import TimeConvert as tc |
| 14 | 16 |
|
@@ -20,7 +22,8 @@ from utils.error.errno_utils import LensmanStatusCode, PhotoStatusCode |
||
| 20 | 22 |
from utils.error.response_utils import response |
| 21 | 23 |
from utils.ip_utils import ip_addr |
| 22 | 24 |
from utils.redis.rgroup import get_group_info, get_group_users_info, set_group_info, set_group_users_info |
| 23 |
-from utils.redis.rkeys import GROUP_LAST_PHOTO_PK |
|
| 25 |
+from utils.redis.rkeys import GROUP_LAST_PHOTO_PK, UUID_LIST |
|
| 26 |
+from utils.redis.ruuid import generate_uuids, update_uuids |
|
| 24 | 27 |
from utils.thumbnail_utils import make_thumbnail |
| 25 | 28 |
from utils.watermark_utils import watermark_wrap |
| 26 | 29 |
|
@@ -37,8 +40,8 @@ def uuid_init(request): |
||
| 37 | 40 |
""" |
| 38 | 41 |
num = int(request.GET.get('num', 1000))
|
| 39 | 42 |
|
| 40 |
- for i in xrange(num): |
|
| 41 |
- UUIDInfo.objects.create(uuid=CurtailUUID.uuid(UUIDInfo)) |
|
| 43 |
+ # 生成 UUID |
|
| 44 |
+ generate_uuids(num) |
|
| 42 | 45 |
|
| 43 | 46 |
return response(200, 'UUID Refresh Success', u'UUID 更新成功') |
| 44 | 47 |
|
@@ -55,13 +58,18 @@ def uuid(request): |
||
| 55 | 58 |
lensman_id = request.POST.get('user_id', '')
|
| 56 | 59 |
num = int(request.POST.get('num', 100))
|
| 57 | 60 |
|
| 58 |
- uuids = UUIDInfo.objects.select_for_update().filter(status=True)[:num] |
|
| 59 |
- for uuid in uuids: |
|
| 60 |
- uuid.lensman_id = lensman_id |
|
| 61 |
- uuid.status = False |
|
| 62 |
- uuid.save() |
|
| 61 |
+ # 从 Redis 中 Pop 中指定数量的 UUID |
|
| 62 |
+ uuids, succeed, left = multi_pop(r, UUID_LIST, num) |
|
| 63 | 63 |
|
| 64 |
- return response(200, 'Get UUID Success', u'获取唯一标识成功', [uuid.uuid for uuid in uuids]) |
|
| 64 |
+ # 异步更新 UUID 数据库中状态 |
|
| 65 |
+ if uuids: |
|
| 66 |
+ async(update_uuids, lensman_id, uuids) |
|
| 67 |
+ |
|
| 68 |
+ # 当可用 UUID 数量少于 500 时, 异步创建 |
|
| 69 |
+ if left < 500: |
|
| 70 |
+ async(generate_uuids) |
|
| 71 |
+ |
|
| 72 |
+ return response(200, 'Get UUID Success', u'获取唯一标识成功', uuids) |
|
| 65 | 73 |
|
| 66 | 74 |
|
| 67 | 75 |
# [How to do a PUT request with curl?](http://stackoverflow.com/questions/13782198/how-to-do-a-put-request-with-curl) |
+ 2
- 0
requirements.txt
@@ -22,7 +22,9 @@ pep8==1.6.2 |
||
| 22 | 22 |
pytz==2015.7 |
| 23 | 23 |
records==0.4.3 |
| 24 | 24 |
redis==2.10.5 |
| 25 |
+redis-extensions==1.0.1 |
|
| 25 | 26 |
shortuuid==0.4.2 |
| 26 | 27 |
uWSGI==2.0.11.1 |
| 27 | 28 |
versions==0.10.0 |
| 28 | 29 |
wechatpy==1.2.8 |
| 30 |
+https://github.com/Brightcells/django-q/archive/master.zip |
+ 3
- 0
utils/redis/rkeys.py
@@ -1,5 +1,8 @@ |
||
| 1 | 1 |
# -*- coding: utf-8 -*- |
| 2 | 2 |
|
| 3 |
+# 唯一标识相关 |
|
| 4 |
+UUID_LIST = 'uuid:list' # List, 唯一标识列表 |
|
| 5 |
+ |
|
| 3 | 6 |
# 用户相关 |
| 4 | 7 |
PROFILE_INFO = 'profile:info:%s' # STRING，用户信息，user_id |
| 5 | 8 |
|
+ 25
- 0
utils/redis/ruuid.py
@@ -0,0 +1,25 @@ |
||
| 1 |
+# -*- coding: utf-8 -*- |
|
| 2 |
+ |
|
| 3 |
+from curtail_uuid import CurtailUUID |
|
| 4 |
+from django.conf import settings |
|
| 5 |
+ |
|
| 6 |
+from photo.models import UUIDInfo |
|
| 7 |
+from utils.redis.rkeys import UUID_LIST |
|
| 8 |
+ |
|
| 9 |
+ |
|
| 10 |
+r = settings.REDIS_CACHE |
|
| 11 |
+ |
|
| 12 |
+ |
|
| 13 |
+def generate_uuid(): |
|
| 14 |
+ uuid = CurtailUUID.uuid(UUIDInfo) |
|
| 15 |
+ UUIDInfo.objects.create(uuid=uuid) |
|
| 16 |
+ return uuid |
|
| 17 |
+ |
|
| 18 |
+ |
|
| 19 |
+def generate_uuids(num=1000): |
|
| 20 |
+ uuids = [generate_uuid() for i in xrange(num)] |
|
| 21 |
+ r.rpush(UUID_LIST, *uuids) |
|
| 22 |
+ |
|
| 23 |
+ |
|
| 24 |
+def update_uuids(lensman_id, uuids): |
|
| 25 |
+ UUIDInfo.objects.filter(uuid__in=uuids).update(lensman_id=lensman_id, status=False) |