一致性hash及php实现（通俗易懂）

  1 <?php
  2 /**
  3  * Flexihash - A simple consistent hashing implementation for PHP.
  4  * 
  5  * The MIT License
  6  * 
  7  * Copyright (c) 2008 Paul Annesley
  8  * 
  9  * Permission is hereby granted, free of charge, to any person obtaining a copy
 10  * of this software and associated documentation files (the "Software"), to deal
 11  * in the Software without restriction, including without limitation the rights
 12  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 13  * copies of the Software, and to permit persons to whom the Software is
 14  * furnished to do so, subject to the following conditions:
 15  * 
 16  * The above copyright notice and this permission notice shall be included in
 17  * all copies or substantial portions of the Software.
 18  * 
 19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 20  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 21  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 22  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 23  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 24  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 25  * THE SOFTWARE.
 26  * 
 27  * @author Paul Annesley
 28  * @link http://paul.annesley.cc/
 29  * @copyright Paul Annesley, 2008
 30  * @comment by MyZ (http://blog.csdn.net/mayongzhan)
 31  */
 32 
 33 /**
 34  * A simple consistent hashing implementation with pluggable hash algorithms.
 35  *
 36  * @author Paul Annesley
 37  * @package Flexihash
 38  * @licence http://www.opensource.org/licenses/mit-license.php
 39  */
 40 class Flexihash
 41 {   
 42     /**
 43      * The number of positions to hash each target to.
 44      *
 45      * @var int
 46      * @comment 虚拟节点数,解决节点分布不均的问题
 47      */
 48     private $_replicas = 64;
 49 
 50     /**
 51      * The hash algorithm, encapsulated in a Flexihash_Hasher implementation.
 52      * @var object Flexihash_Hasher
 53      * @comment 使用的hash方法 : md5,crc32
 54      */
 55     private $_hasher;
 56 
 57     /**
 58      * Internal counter for current number of targets.
 59      * @var int
 60      * @comment 节点记数器
 61      */
 62     private $_targetCount = 0;
 63 
 64     /**
 65      * Internal map of positions (hash outputs) to targets
 66      * @var array { position => target, ... }
 67      * @comment 位置对应节点,用于lookup中根据位置确定要访问的节点
 68      */
 69     private $_positionToTarget = array();
 70 
 71     /**
 72      * Internal map of targets to lists of positions that target is hashed to.
 73      * @var array { target => [ position, position, ... ], ... }
 74      * @comment 节点对应位置,用于删除节点
 75      */
 76     private $_targetToPositions = array();
 77 
 78     /**
 79      * Whether the internal map of positions to targets is already sorted.
 80      * @var boolean
 81      * @comment 是否已排序
 82      */
 83     private $_positionToTargetSorted = false;
 84 
 85     /**
 86      * Constructor
 87      * @param object $hasher Flexihash_Hasher
 88      * @param int $replicas Amount of positions to hash each target to.
 89      * @comment 构造函数,确定要使用的hash方法和需拟节点数,虚拟节点数越多,分布越均匀,但程序的分布式运算越慢
 90      */
 91     public function __construct(Flexihash_Hasher $hasher = null, $replicas = null)
 92     {
 93         $this->_hasher = $hasher ? $hasher : new Flexihash_Crc32Hasher();
 94         if (!empty($replicas)) $this->_replicas = $replicas;
 95     }
 96 
 97     /**
 98      * Add a target.
 99      * @param string $target
100      * @chainable
101      * @comment 添加节点,根据虚拟节点数,将节点分布到多个虚拟位置上
102      */
103     public function addTarget($target)
104     {
105         if (isset($this->_targetToPositions[$target]))
106         {
107             throw new Flexihash_Exception("Target '$target' already exists.");
108         }
109 
110         $this->_targetToPositions[$target] = array();
111 
112         // hash the target into multiple positions
113         for ($i = 0; $i < $this->_replicas; $i++)
114         {
115             $position = $this->_hasher->hash($target . $i);
116             $this->_positionToTarget[$position] = $target; // lookup
117             $this->_targetToPositions[$target] []= $position; // target removal
118         }
119 
120         $this->_positionToTargetSorted = false;
121         $this->_targetCount++;
122 
123         return $this;
124     }
125 
126     /**
127      * Add a list of targets.
128      * @param array $targets
129      * @chainable
130      */
131     public function addTargets($targets)
132     {
133         foreach ($targets as $target)
134         {
135             $this->addTarget($target);
136         }
137 
138         return $this;
139     }
140 
141     /**
142      * Remove a target.
143      * @param string $target
144      * @chainable
145      */
146     public function removeTarget($target)
147     {
148         if (!isset($this->_targetToPositions[$target]))
149         {
150             throw new Flexihash_Exception("Target '$target' does not exist.");
151         }
152 
153         foreach ($this->_targetToPositions[$target] as $position)
154         {
155             unset($this->_positionToTarget[$position]);
156         }
157 
158         unset($this->_targetToPositions[$target]);
159 
160         $this->_targetCount--;
161 
162         return $this;
163     }
164 
165     /**
166      * A list of all potential targets
167      * @return array
168      */
169     public function getAllTargets()
170     {
171         return array_keys($this->_targetToPositions);
172     }
173 
174     /**
175      * A list of all potential targets
176      * @return array
177      */
178     public function getAll()
179     {
180         return array(
181             "targers"=>$this->_positionToTarget, 
182             "positions"=>$this->_targetToPositions);
183     }
184     
185     /**
186      * Looks up the target for the given resource.
187      * @param string $resource
188      * @return string
189      */
190     public function lookup($resource)
191     {
192         $targets = $this->lookupList($resource, 1);
193         if (empty($targets)) throw new Flexihash_Exception('No targets exist');
194         return $targets[0]; //0表示返回离资源位置最近的机器节点
195     }
196 
197     /**
198      * Get a list of targets for the resource, in order of precedence.
199      * Up to $requestedCount targets are returned, less if there are fewer in total.
200      *
201      * @param string $resource
202      * @param int $requestedCount The length of the list to return
203      * @return array List of targets
204      * @comment 查找当前的资源对应的节点,
205      *          节点为空则返回空,节点只有一个则返回该节点,
206      *          对当前资源进行hash,对所有的位置进行排序,在有序的位置列上寻找当前资源的位置
207      *          当全部没有找到的时候,将资源的位置确定为有序位置的第一个(形成一个环)
208      *          返回所找到的节点
209      */
210     public function lookupList($resource, $requestedCount)
211     {
212         if (!$requestedCount)
213             throw new Flexihash_Exception('Invalid count requested');
214 
215         // handle no targets
216         if (empty($this->_positionToTarget))
217             return array();
218 
219         // optimize single target
220         if ($this->_targetCount == 1)
221             return array_unique(array_values($this->_positionToTarget));
222 
223         // hash resource to a position
224         $resourcePosition = $this->_hasher->hash($resource);
225 
226         $results = array();
227         $collect = false;
228 
229         $this->_sortPositionTargets();
230 
231         // search values above the resourcePosition
232         foreach ($this->_positionToTarget as $key => $value)
233         {
234             // start collecting targets after passing resource position
235             if (!$collect && $key > $resourcePosition)
236             {
237                 $collect = true;
238             }
239 
240             // only collect the first instance of any target
241             if ($collect && !in_array($value, $results))
242             {
243                 $results []= $value;
244                 //var_dump($results);
245             }
246             // return when enough results, or list exhausted
247             //var_dump(count($results));
248             //var_dump($requestedCount);
249             if (count($results) == $requestedCount || count($results) == $this->_targetCount)
250             {
251                 return $results;
252             }
253         }
254 
255         // loop to start - search values below the resourcePosition
256         foreach ($this->_positionToTarget as $key => $value)
257         {
258             if (!in_array($value, $results))
259             {
260                 $results []= $value;
261             }
262 
263             // return when enough results, or list exhausted
264             if (count($results) == $requestedCount || count($results) == $this->_targetCount)
265             {
266                 return $results;
267             }
268         }
269 
270         // return results after iterating through both "parts"
271         return $results;
272     }
273 
274     public function __toString()
275     {
276         return sprintf(
277             '%s{targets:[%s]}',
278             get_class($this),
279             implode(',', $this->getAllTargets())
280         );
281     }
282 
283     // ----------------------------------------
284     // private methods
285 
286     /**
287      * Sorts the internal mapping (positions to targets) by position
288      */
289     private function _sortPositionTargets()
290     {
291         // sort by key (position) if not already
292         if (!$this->_positionToTargetSorted)
293         {
294             ksort($this->_positionToTarget, SORT_REGULAR);
295             $this->_positionToTargetSorted = true;
296         }
297     }
298 
299 }
300 
301 /**
302  * Hashes given values into a sortable fixed size address space.
303  *
304  * @author Paul Annesley
305  * @package Flexihash
306  * @licence http://www.opensource.org/licenses/mit-license.php
307  */
308 interface Flexihash_Hasher
309 {
310 
311     /**
312      * Hashes the given string into a 32bit address space.
313      *
314      * Note that the output may be more than 32bits of raw data, for example
315      * hexidecimal characters representing a 32bit value.
316      *
317      * The data must have 0xFFFFFFFF possible values, and be sortable by
318      * PHP sort functions using SORT_REGULAR.
319      *
320      * @param string
321      * @return mixed A sortable format with 0xFFFFFFFF possible values
322      */
323     public function hash($string);
324 
325 }
326 
327 /**
328  * Uses CRC32 to hash a value into a signed 32bit int address space.
329  * Under 32bit PHP this (safely) overflows into negatives ints.
330  *
331  * @author Paul Annesley
332  * @package Flexihash
333  * @licence http://www.opensource.org/licenses/mit-license.php
334  */
335 class Flexihash_Crc32Hasher
336     implements Flexihash_Hasher
337 {
338 
339     /* (non-phpdoc)
340      * @see Flexihash_Hasher::hash()
341      */
342     public function hash($string)
343     {
344         return crc32($string);
345     }
346 
347 }
348 
349 /**
350  * Uses CRC32 to hash a value into a 32bit binary string data address space.
351  *
352  * @author Paul Annesley
353  * @package Flexihash
354  * @licence http://www.opensource.org/licenses/mit-license.php
355  */
356 class Flexihash_Md5Hasher
357     implements Flexihash_Hasher
358 {
359 
360     /* (non-phpdoc)
361      * @see Flexihash_Hasher::hash()
362      */
363     public function hash($string)
364     {
365         return substr(md5($string), 0, 8); // 8 hexits = 32bit
366 
367         // 4 bytes of binary md5 data could also be used, but
368         // performance seems to be the same.
369     }
370 
371 }
372 
373 /**
374  * An exception thrown by Flexihash.
375  *
376  * @author Paul Annesley
377  * @package Flexihash
378  * @licence http://www.opensource.org/licenses/mit-license.php
379  */
380 class Flexihash_Exception extends Exception
381 {
382 }
383 
384 测试代码
385 $hash = new Flexihash();
386 $targets=array(
387     "192.168.1.1:11011",
388     "192.168.1.1:11012",
389     "192.168.1.1:11013",
390     "192.168.1.1:11014",
391     "192.168.1.1:11015",
392 );
393 $hash->addTargets($targets);
394 for ($i=0; $i < 25; $i++) {
395     $resource = sprintf("format %d",$i);
396     var_dump($resource." --> ".$hash->lookup($resource));
397 }
398 
399 输出
400 
401 string(30) "format 0 --> 192.168.1.1:11015"
402 string(30) "format 1 --> 192.168.1.1:11015"
403 string(30) "format 2 --> 192.168.1.1:11015"
404 string(30) "format 3 --> 192.168.1.1:11015"
405 string(30) "format 4 --> 192.168.1.1:11011"
406 string(30) "format 5 --> 192.168.1.1:11011"
407 string(30) "format 6 --> 192.168.1.1:11011"
408 string(30) "format 7 --> 192.168.1.1:11011"
409 string(30) "format 8 --> 192.168.1.1:11012"
410 string(30) "format 9 --> 192.168.1.1:11013"
411 string(31) "format 10 --> 192.168.1.1:11013"
412 string(31) "format 11 --> 192.168.1.1:11011"
413 string(31) "format 12 --> 192.168.1.1:11012"
414 string(31) "format 13 --> 192.168.1.1:11011"
415 string(31) "format 14 --> 192.168.1.1:11014"
416 string(31) "format 15 --> 192.168.1.1:11014"
417 string(31) "format 16 --> 192.168.1.1:11014"
418 string(31) "format 17 --> 192.168.1.1:11014"
419 string(31) "format 18 --> 192.168.1.1:11012"
420 string(31) "format 19 --> 192.168.1.1:11012"
421 string(31) "format 20 --> 192.168.1.1:11013"
422 string(31) "format 21 --> 192.168.1.1:11012"
423 string(31) "format 22 --> 192.168.1.1:11012"
424 string(31) "format 23 --> 192.168.1.1:11014"
425 string(31) "format 24 --> 192.168.1.1:11012"
426 [Finished in 0.1s]
427 
428 redis分布式代码设计
429 
430 <?php
431 require_once("Flexihash.php");
432 $config=array(
433     "127.0.0.1:6371",
434     "127.0.0.1:6372",
435     "127.0.0.1:6373",
436     "127.0.0.1:6374",
437 );
438 class RedisCollect {
439     //redis实例
440     private $_redis = null;
441     //hash实例
442     private $_hash = null;
443     //初始化
444     public function __construct() {
445         global $config;
446         $this->_redis = new Redis();
447         $this->_hash = new Flexihash();
448         $this->_hash->addTargets($config);
449     }
450     public function set($key="", $value="") {
451         $m = $this->switchConncetion($key);
452         return $m->set($key, $value);
453     }
454     public function get($key) {
455         $m = $this->switchConncetion($key);
456         return $m->get($key);
457     }
458     private function switchConncetion($key) {
459         $hostinfo = $this->_hash->lookup($key);
460         $m = $this->connect($hostinfo);
461         return $m;
462     }
463     private function connect($hostinfo) {
464         list($host, $port) = explode(":", $hostinfo);
465         //printf("host = %s, port = %s\n",$host,$port);
466         if(empty($host) || empty($port)) {
467             return false;
468         }
469         try {
470             $this->_redis->connect($host, $port);
471             return $this->_redis;
472         } catch(Exception $e) {
473             die($e->getMessage());
474         }
475     }
476 }