From 8395074dde44760a334fe5b00354340165aaf695 Mon Sep 17 00:00:00 2001 From: xiaochong0302 Date: Mon, 23 Dec 2019 22:43:04 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E7=B4=A2=E5=BC=95=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E6=94=B9=E7=94=A8=E8=AE=A1=E5=88=92=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E6=89=B9=E9=87=8F=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + ...ourseCountTask.php => CountCourseTask.php} | 2 +- app/Console/Tasks/LearningTask.php | 25 ++- app/Console/Tasks/ManageIndexTask.php | 166 ++++++++++++++++++ app/Console/Tasks/SyncIndexTask.php | 66 +++++++ app/Http/Admin/Services/Course.php | 8 + app/Library/Cache/Backend/Redis.php | 23 ++- app/Listeners/Course.php | 124 +++++++++++++ app/Providers/Cache.php | 1 + app/Providers/Session.php | 1 + app/Searchers/Course.php | 98 ++++++----- ...{config.default.php => config.php.default} | 20 ++- config/events.php | 1 + public/static/admin/js/common.js | 2 +- scheduler.php | 9 +- 15 files changed, 475 insertions(+), 72 deletions(-) rename app/Console/Tasks/{CourseCountTask.php => CountCourseTask.php} (96%) create mode 100644 app/Console/Tasks/ManageIndexTask.php create mode 100644 app/Console/Tasks/SyncIndexTask.php create mode 100644 app/Listeners/Course.php rename config/{config.default.php => config.php.default} (82%) diff --git a/.gitignore b/.gitignore index 9e094c8c..75e4ead1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /vendor /config/config.php /config/xs.course.ini +*KgTest* diff --git a/app/Console/Tasks/CourseCountTask.php b/app/Console/Tasks/CountCourseTask.php similarity index 96% rename from app/Console/Tasks/CourseCountTask.php rename to app/Console/Tasks/CountCourseTask.php index f522cdc3..6c065d0d 100644 --- a/app/Console/Tasks/CourseCountTask.php +++ b/app/Console/Tasks/CountCourseTask.php @@ -5,7 +5,7 @@ namespace App\Console\Tasks; use App\Repos\Category as CategoryRepo; use Phalcon\Cli\Task; -class CourseCountTask extends Task +class CountCourseTask extends Task { public function mainAction() diff --git a/app/Console/Tasks/LearningTask.php b/app/Console/Tasks/LearningTask.php index ce874343..be1918df 100644 --- a/app/Console/Tasks/LearningTask.php +++ b/app/Console/Tasks/LearningTask.php @@ -24,20 +24,15 @@ class LearningTask extends Task $keys = $this->cache->queryKeys('learning:'); - if (empty($keys)) return; + if (empty($keys)) { + return; + } $keys = array_slice($keys, 0, 500); - $prefix = $this->cache->getPrefix(); - foreach ($keys as $key) { - /** - * 去掉前缀，避免重复加前缀导致找不到缓存 - */ - if ($prefix) { - $key = str_replace($prefix, '', $key); - } - $this->handleLearning($key); + $lastKey = $this->cache->getRawKeyName($key); + $this->handleLearning($lastKey); } } @@ -46,7 +41,7 @@ class LearningTask extends Task $content = $this->cache->get($key); if (empty($content->user_id)) { - return false; + return; } if (!empty($content->client_ip)) { @@ -81,13 +76,17 @@ class LearningTask extends Task $chapterUser = $chapterUserRepo->findChapterUser($chapterId, $userId); - if (!$chapterUser) return false; + if (!$chapterUser) { + return; + } $chapterRepo = new ChapterRepo(); $chapter = $chapterRepo->findById($chapterId); - if (!$chapter) return false; + if (!$chapter) { + return; + } $chapter->duration = $chapter->attrs['duration'] ?: 0; diff --git a/app/Console/Tasks/ManageIndexTask.php b/app/Console/Tasks/ManageIndexTask.php new file mode 100644 index 00000000..edaeafcb --- /dev/null +++ b/app/Console/Tasks/ManageIndexTask.php @@ -0,0 +1,166 @@ +getItemTypes())) { + exit("Invalid item type" . PHP_EOL); + } + + if (!$query) { + exit("Please special a query word" . PHP_EOL); + } + + $result = []; + + if ($type == 'course') { + $result = $this->searchCourses($query); + } + + var_export($result); + } + + /** + * 清空索引 + * + * @command: php console.php index clean {type} + * @param array $params + */ + public function cleanAction($params) + { + $type = $params[0] ?? null; + + if (in_array($type, $this->getItemTypes())) { + exit("Invalid item type" . PHP_EOL); + } + + if ($type == 'course') { + $this->cleanCourseIndex(); + } + } + + /** + * 重建索引 + * + * @command: php console.php index rebuild {type} + * @param array $params + */ + public function rebuildAction($params) + { + $type = $params[0] ?? null; + + if (in_array($type, $this->getItemTypes())) { + exit("Invalid item type" . PHP_EOL); + } + + if ($type == 'course') { + $this->rebuildCourseIndex(); + } + } + + /** + * 清空课程索引 + */ + protected function cleanCourseIndex() + { + $searcher = new CourseSearcher(); + + $index = $searcher->getXS()->getIndex(); + + echo "Start clean index" . PHP_EOL; + + $index->clean(); + + echo "End clean index" . PHP_EOL; + } + + /** + * 重建课程索引 + */ + protected function rebuildCourseIndex() + { + $courses = $this->findCourses(); + + if ($courses->count() == 0) { + return; + } + + $searcher = new CourseSearcher(); + + $index = $searcher->getXS()->getIndex(); + + echo "Start rebuild index" . PHP_EOL; + + $index->beginRebuild(); + + foreach ($courses as $course) { + $document = $searcher->setDocument($course); + $index->add($document); + } + + $index->endRebuild(); + + echo "End rebuild index" . PHP_EOL; + } + + /** + * 搜索课程 + * + * @param string $query + * @return \stdClass $result + * @throws \XSException + */ + protected function searchCourses($query) + { + $searcher = new CourseSearcher(); + + $result = $searcher->search($query); + + return $result; + } + + /** + * 查找课程 + * + * @return \Phalcon\Mvc\Model\ResultsetInterface + */ + protected function findCourses() + { + $courses = CourseModel::query() + ->where('published = 1') + ->execute(); + + return $courses; + } + + /** + * 获取条目类型 + * + * @return array + */ + protected function getItemTypes() + { + $types = ['course']; + + return $types; + } + +} diff --git a/app/Console/Tasks/SyncIndexTask.php b/app/Console/Tasks/SyncIndexTask.php new file mode 100644 index 00000000..e6ce5133 --- /dev/null +++ b/app/Console/Tasks/SyncIndexTask.php @@ -0,0 +1,66 @@ +cache = $this->getDI()->get('cache'); + + $this->handleCourseIndex(); + } + + protected function handleCourseIndex() + { + $keys = $this->cache->queryKeys('sync:index:course:'); + + if (empty($keys)) { + return; + } + + $keys = array_slice($keys, 0, 100); + + $searcher = new CourseSearcher(); + + $index = $searcher->getXS()->getIndex(); + + $index->openBuffer(); + + foreach ($keys as $key) { + + $lastKey = $this->cache->getRawKeyName($key); + $content = $this->cache->get($lastKey); + + if (empty($content->id)) { + continue; + } + + $course = CourseModel::findFirstById($content->id); + + $document = $searcher->setDocument($course); + + if ($content->type == 'update') { + $index->update($document); + } elseif ($content->type == 'delete') { + $index->del($course->id); + } elseif ($content->type == 'restore') { + $index->update($document); + } + + $this->cache->delete($lastKey); + } + + $index->closeBuffer(); + } + +} diff --git a/app/Http/Admin/Services/Course.php b/app/Http/Admin/Services/Course.php index ec28f24f..66b33b61 100644 --- a/app/Http/Admin/Services/Course.php +++ b/app/Http/Admin/Services/Course.php @@ -66,6 +66,8 @@ class Course extends Service $course->create($data); + $this->eventsManager->fire('course:afterCreate', $this, $course); + return $course; } @@ -135,6 +137,8 @@ class Course extends Service $course->update($data); + $this->eventsManager->fire('course:afterUpdate', $this, $course); + return $course; } @@ -150,6 +154,8 @@ class Course extends Service $course->update(); + $this->eventsManager->fire('course:afterDelete', $this, $course); + return $course; } @@ -165,6 +171,8 @@ class Course extends Service $course->update(); + $this->eventsManager->fire('course:afterRestore', $this, $course); + return $course; } diff --git a/app/Library/Cache/Backend/Redis.php b/app/Library/Cache/Backend/Redis.php index 954f8395..560e2d92 100644 --- a/app/Library/Cache/Backend/Redis.php +++ b/app/Library/Cache/Backend/Redis.php @@ -224,7 +224,7 @@ class Redis extends \Phalcon\Cache\Backend\Redis } /** - * Get Prefix + * Get prefix * * @return string */ @@ -234,7 +234,7 @@ class Redis extends \Phalcon\Cache\Backend\Redis } /** - * Get Redis Connection + * Get redis connection * * @return \Redis */ @@ -251,14 +251,29 @@ class Redis extends \Phalcon\Cache\Backend\Redis } /** - * Get Key Name + * Get key name * * @param $keyName * @return string */ - protected function getKeyName($keyName) + public function getKeyName($keyName) { return $this->_prefix . $keyName; } + /** + * Get raw key name + * + * @param $keyName + * @return string + */ + public function getRawKeyName($keyName) + { + if ($this->_prefix) { + $keyName = str_replace($this->_prefix, '', $keyName); + } + + return $keyName; + } + } diff --git a/app/Listeners/Course.php b/app/Listeners/Course.php new file mode 100644 index 00000000..71259c50 --- /dev/null +++ b/app/Listeners/Course.php @@ -0,0 +1,124 @@ +logger = $this->getLogger(); + } + + public function afterCreate(Event $event, $source, CourseModel $course) + { + $this->logger->debug('Event: {event}, Source: {source}, Data: {data}', [ + 'event' => $event->getType(), + 'source' => get_class($source), + 'data' => kg_json_encode($course), + ]); + } + + public function afterUpdate(Event $event, $source, CourseModel $course) + { + if ($course->published == 1) { + $this->syncIndexAfterUpdate($course); + } + + $this->logger->debug('Event: {event}, Source: {source}, Data: {data}', [ + 'event' => $event->getType(), + 'source' => get_class($source), + 'data' => kg_json_encode($course), + ]); + } + + public function afterDelete(Event $event, $source, CourseModel $course) + { + if ($course->published == 1) { + $this->syncIndexAfterDelete($course); + } + + $this->logger->debug('Event: {event}, Source: {source}, Data: {data}', [ + 'event' => $event->getType(), + 'source' => get_class($source), + 'data' => kg_json_encode($course), + ]); + } + + public function afterRestore(Event $event, $source, CourseModel $course) + { + if ($course->published == 1) { + $this->syncIndexAfterRestore($course); + } + + $this->logger->debug('Event: {event}, Source: {source}, Data: {data}', [ + 'event' => $event->getType(), + 'source' => get_class($source), + 'data' => kg_json_encode($course), + ]); + } + + protected function syncIndexAfterUpdate(CourseModel $course) + { + /** + * @var \Phalcon\Cache\Backend $cache + */ + $cache = $this->getDI()->get('cache'); + + $key = $this->getSyncIndexKey($course->id); + + $content = [ + 'id' => $course->id, + 'type' => 'update', + ]; + + $cache->save($key, $content, 86400); + } + + protected function syncIndexAfterDelete($course) + { + /** + * @var \Phalcon\Cache\Backend $cache + */ + $cache = $this->getDI()->get('cache'); + + $key = $this->getSyncIndexKey($course->id); + + $content = [ + 'id' => $course->id, + 'type' => 'delete', + ]; + + $cache->save($key, $content, 86400); + } + + protected function syncIndexAfterRestore($course) + { + /** + * @var \Phalcon\Cache\Backend $cache + */ + $cache = $this->getDI()->get('cache'); + + $key = $this->getSyncIndexKey($course->id); + + $content = [ + 'id' => $course->id, + 'type' => 'restore', + ]; + + $cache->save($key, $content, 86400); + } + + protected function getSyncIndexKey($courseId) + { + $key = "sync:index:course:{$courseId}"; + + return $key; + } + +} \ No newline at end of file diff --git a/app/Providers/Cache.php b/app/Providers/Cache.php index 03521329..6eddc179 100644 --- a/app/Providers/Cache.php +++ b/app/Providers/Cache.php @@ -25,6 +25,7 @@ class Cache extends AbstractProvider 'port' => $config->redis->port, 'auth' => $config->redis->auth, 'index' => $config->redis->index, + 'prefix' => $config->redis->prefix, 'persistent' => $config->redis->persistent, ]); diff --git a/app/Providers/Session.php b/app/Providers/Session.php index 7405b989..954641cc 100644 --- a/app/Providers/Session.php +++ b/app/Providers/Session.php @@ -20,6 +20,7 @@ class Session extends AbstractProvider 'port' => $config->redis->port, 'auth' => $config->redis->auth, 'index' => $config->session->index, + 'prefix' => $config->session->prefix, 'lifetime' => $config->session->lifetime, 'persistent' => $config->redis->persistent, ]); diff --git a/app/Searchers/Course.php b/app/Searchers/Course.php index 653980a9..19384c59 100644 --- a/app/Searchers/Course.php +++ b/app/Searchers/Course.php @@ -8,7 +8,10 @@ use Phalcon\Mvc\User\Component as UserComponent; class Course extends UserComponent { - private $xs; + /** + * @var \XS + */ + protected $xs; public function __construct() { @@ -18,61 +21,53 @@ class Course extends UserComponent } /** - * 搜索 + * 获取XS + * @return \XS + */ + public function getXS() + { + return $this->xs; + } + + /** + * 搜索课程 * * @param string $query * @param integer $limit * @param integer $offset + * @return \stdClass * @throws \XSException - * @return array */ public function search($query, $limit = 15, $offset = 0) { - $search = $this->xs->search; + $search = $this->xs->getSearch(); - $items = $search->setQuery($query)->setLimit($limit, $offset)->search(); + $docs = $search->setQuery($query)->setLimit($limit, $offset)->search(); $total = $search->getLastCount(); - return [ - 'total' => $total, - 'items' => $items, - ]; - } + $fields = array_keys($this->xs->getAllFields()); - /** - * 添加索引 - * - * @param CourseModel $course - */ - public function addIndex($course) - { - $doc = $this->setXSDocument($course); + $items = []; - $this->xs->index->add($doc); - } + foreach ($docs as $doc) { + $item = new \stdClass(); + foreach ($fields as $field) { + if (in_array($field, ['title', 'summary'])) { + $item->{$field} = $search->highlight($doc->{$field}); + } else { + $item->{$field} = $doc->{$field}; + } + } + $items[] = $item; + } - /** - * 更新索引 - * - * @param CourseModel $course - * @throws \XSException - */ - public function updateIndex($course) - { - $doc = $this->setXSDocument($course); + $result = new \stdClass(); - $this->xs->index->update($doc); - } + $result->total = $total; + $result->items = $items; - /** - * 删除索引 - * - * @param CourseModel $course - */ - public function deleteIndex($course) - { - $this->xs->index->del($course->id); + return $result; } /** @@ -81,7 +76,24 @@ class Course extends UserComponent * @param CourseModel $course * @return \XSDocument */ - private function setXSDocument($course) + public function setDocument(CourseModel $course) + { + $doc = new \XSDocument(); + + $data = $this->formatDocument($course); + + $doc->setFields($data); + + return $doc; + } + + /** + * 格式化文档 + * + * @param CourseModel $course + * @return array + */ + public function formatDocument(CourseModel $course) { $data = [ 'id' => $course->id, @@ -101,11 +113,7 @@ class Course extends UserComponent 'created_at' => $course->created_at, ]; - $doc = new \XSDocument(); - - $doc->setFields($data); - - return $doc; + return $data; } } diff --git a/config/config.default.php b/config/config.php.default similarity index 82% rename from config/config.default.php rename to config/config.php.default index 5f64c8b3..5ab4a195 100644 --- a/config/config.default.php +++ b/config/config.php.default @@ -73,7 +73,7 @@ $config['redis']['port'] = 6379; $config['redis']['auth'] = '1qaz2wsx3edc'; /** - * 数据库编号 + * redis库编号 */ $config['redis']['index'] = 0; @@ -83,17 +83,27 @@ $config['redis']['index'] = 0; $config['redis']['persistent'] = false; /** - * 默认有限期（秒） + * redis键前缀 */ -$config['redis']['lifetime'] = 86400; +$config['redis']['prefix'] = ''; + +/** + * redis有限期（秒） + */ +$config['redis']['lifetime'] = 7 * 86400; + +/** + * 会话键前缀 + */ +$config['session']['prefix'] = ''; /** * 会话有效期（秒） */ -$config['session']['lifetime'] = 7200; +$config['session']['lifetime'] = 2 * 3600; /** - * 数据库编号 + * redis库编号 */ $config['session']['index'] = 1; diff --git a/config/events.php b/config/events.php index e3d6e97e..bd223f78 100644 --- a/config/events.php +++ b/config/events.php @@ -3,6 +3,7 @@ $events = [ 'db' => \App\Listeners\Profiler::class, + 'course' => \App\Listeners\Course::class, 'payment' => \App\Listeners\Payment::class, ]; diff --git a/public/static/admin/js/common.js b/public/static/admin/js/common.js index 4249a26c..ae0e4e0b 100644 --- a/public/static/admin/js/common.js +++ b/public/static/admin/js/common.js @@ -27,8 +27,8 @@ layui.use(['jquery', 'form', 'element', 'layer', 'dropdown'], function () { url: data.form.action, data: data.field, success: function (res) { + var icon = res.code == 0 ? 1 : 2; if (res.msg != '') { - var icon = (res.code == 0) ? 1 : 2; layer.msg(res.msg, {icon: icon}); } if (res.location) { diff --git a/scheduler.php b/scheduler.php index af9d425d..3793c6f6 100644 --- a/scheduler.php +++ b/scheduler.php @@ -16,18 +16,21 @@ $scheduler->php($script, $bin, ['--task' => 'close_trade', '--action' => 'main'] ->at('*/15 * * * *'); $scheduler->php($script, $bin, ['--task' => 'close_order', '--action' => 'main']) - ->at('* */6 * * *'); + ->at('* */3 * * *'); $scheduler->php($script, $bin, ['--task' => 'learning', '--action' => 'main']) ->at('*/10 * * * *'); $scheduler->php($script, $bin, ['--task' => 'refund', '--action' => 'main']) - ->hourly(15); + ->hourly(3); + +$scheduler->php($script, $bin, ['--task' => 'sync_index', '--action' => 'main']) + ->hourly(13); $scheduler->php($script, $bin, ['--task' => 'clean_log', '--action' => 'main']) ->daily(3, 10); -$scheduler->php($script, $bin, ['--task' => 'course_count', '--action' => 'main']) +$scheduler->php($script, $bin, ['--task' => 'count_course', '--action' => 'main']) ->daily(3, 20); $scheduler->php($script, $bin, ['--task' => 'unlock_user', '--action' => 'main'])