mirrors/system-design-primer
1
0
Fork 0
mirror of https://github.com/donnemartin/system-design-primer.git synced 2025-09-17 09:30:39 +03:00
Code Issues Projects Releases Wiki Activity

poriting to noat.cards

Browse Source
This commit is contained in:
Vu
2021-03-14 17:08:05 +07:00
parent 6984b4e956
commit f4af06bdff
48 changed files with 3545 additions and 3384 deletions
Download Patch File Download Diff File Expand all files Collapse all files

172
solutions/system_design/social_graph/README-zh-Hans.md
View File

@@ -29,7 +29,7 @@
* 每个用户平均有 50 个朋友
* 每月 10 亿次朋友搜索
训练使用更传统的系统 - 别用图特有的解决方案例如 [GraphQL](http://graphql.org/) 或图数据库如 [Neo4j](https://neo4j.com/)。
训练使用更传统的系统 - 别用图特有的解决方案例如 [GraphQL](http://graphql.org/) 或图数据库如 [Neo4j](https://neo4j.com/) 。
#### 计算使用
@@ -50,7 +50,7 @@
> 用所有重要组件概述高水平设计
![Imgur](http://i.imgur.com/wxXyq2J.png)
![Imgur](http://i.imgur.com/wxXyq2J.png)
## 第 3 步：设计核心组件
@@ -63,37 +63,37 @@
没有百万用户（点）的和十亿朋友关系（边）的限制，我们能够用一般 BFS 方法解决无权重最短路径任务：
```python
class Graph(Graph):
class Graph(Graph) :
def shortest_path(self, source, dest):
def shortest_path(self, source, dest) :
if source is None or dest is None:
return None
if source is dest:
return [source.key]
prev_node_keys = self._shortest_path(source, dest)
prev_node_keys = self._shortest_path(source, dest)
if prev_node_keys is None:
return None
else:
path_ids = [dest.key]
prev_node_key = prev_node_keys[dest.key]
while prev_node_key is not None:
path_ids.append(prev_node_key)
path_ids.append(prev_node_key)
prev_node_key = prev_node_keys[prev_node_key]
return path_ids[::-1]
def _shortest_path(self, source, dest):
queue = deque()
queue.append(source)
def _shortest_path(self, source, dest) :
queue = deque()
queue.append(source)
prev_node_keys = {source.key: None}
source.visit_state = State.visited
while queue:
node = queue.popleft()
node = queue.popleft()
if node is dest:
return prev_node_keys
prev_node = node
for adj_node in node.adj_nodes.values():
for adj_node in node.adj_nodes.values() :
if adj_node.visit_state == State.unvisited:
queue.append(adj_node)
queue.append(adj_node)
prev_node_keys[adj_node.key] = prev_node.key
adj_node.visit_state = State.visited
return None
@@ -101,7 +101,7 @@ class Graph(Graph):
我们不能在同一台机器上满足所有用户，我们需要通过 **人员服务器** [拆分](https://github.com/donnemartin/system-design-primer#sharding) 用户并且通过 **查询服务** 访问。
* **客户端** 向 **服务器** 发送请求，**服务器** 作为 [反向代理](https://github.com/donnemartin/system-design-primer#reverse-proxy-web-server)
* **客户端** 向 **服务器** 发送请求，**服务器** 作为 [反向代理](https://github.com/donnemartin/system-design-primer#reverse-proxy-web-server)
* **搜索 API** 服务器向 **用户图服务** 转发请求
* **用户图服务** 有以下功能：
* 使用 **查询服务** 找到当前用户信息存储的 **人员服务器**
@@ -117,43 +117,43 @@ class Graph(Graph):
**查询服务** 实现：
```python
class LookupService(object):
class LookupService(object) :
def __init__(self):
self.lookup = self._init_lookup() # key: person_id, value: person_server
def __init__(self) :
self.lookup = self._init_lookup() # key: person_id, value: person_server
def _init_lookup(self):
def _init_lookup(self) :
...
def lookup_person_server(self, person_id):
def lookup_person_server(self, person_id) :
return self.lookup[person_id]
```
**人员服务器** 实现：
```python
class PersonServer(object):
class PersonServer(object) :
def __init__(self):
def __init__(self) :
self.people = {} # key: person_id, value: person
def add_person(self, person):
def add_person(self, person) :
...
def people(self, ids):
def people(self, ids) :
results = []
for id in ids:
if id in self.people:
results.append(self.people[id])
results.append(self.people[id])
return results
```
**用户** 实现：
```python
class Person(object):
class Person(object) :
def __init__(self, id, name, friend_ids):
def __init__(self, id, name, friend_ids) :
self.id = id
self.name = name
self.friend_ids = friend_ids
@@ -162,21 +162,21 @@ class Person(object):
**用户图服务** 实现：
```python
class UserGraphService(object):
class UserGraphService(object) :
def __init__(self, lookup_service):
def __init__(self, lookup_service) :
self.lookup_service = lookup_service
def person(self, person_id):
person_server = self.lookup_service.lookup_person_server(person_id)
return person_server.people([person_id])
def person(self, person_id) :
person_server = self.lookup_service.lookup_person_server(person_id)
return person_server.people([person_id])
def shortest_path(self, source_key, dest_key):
def shortest_path(self, source_key, dest_key) :
if source_key is None or dest_key is None:
return None
if source_key is dest_key:
return [source_key]
prev_node_keys = self._shortest_path(source_key, dest_key)
prev_node_keys = self._shortest_path(source_key, dest_key)
if prev_node_keys is None:
return None
else:
@@ -184,40 +184,40 @@ class UserGraphService(object):
path_ids = [dest_key]
prev_node_key = prev_node_keys[dest_key]
while prev_node_key is not None:
path_ids.append(prev_node_key)
path_ids.append(prev_node_key)
prev_node_key = prev_node_keys[prev_node_key]
# Reverse the list since we iterated backwards
return path_ids[::-1]
def _shortest_path(self, source_key, dest_key, path):
def _shortest_path(self, source_key, dest_key, path) :
# Use the id to get the Person
source = self.person(source_key)
source = self.person(source_key)
# Update our bfs queue
queue = deque()
queue.append(source)
queue = deque()
queue.append(source)
# prev_node_keys keeps track of each hop from
# the source_key to the dest_key
prev_node_keys = {source_key: None}
# We'll use visited_ids to keep track of which nodes we've
# visited, which can be different from a typical bfs where
# this can be stored in the node itself
visited_ids = set()
visited_ids.add(source.id)
visited_ids = set()
visited_ids.add(source.id)
while queue:
node = queue.popleft()
node = queue.popleft()
if node.key is dest_key:
return prev_node_keys
prev_node = node
for friend_id in node.friend_ids:
if friend_id not in visited_ids:
friend_node = self.person(friend_id)
queue.append(friend_node)
friend_node = self.person(friend_id)
queue.append(friend_node)
prev_node_keys[friend_id] = prev_node.key
visited_ids.add(friend_id)
visited_ids.add(friend_id)
return None
```
我们用的是公共的 [**REST API**](https://github.com/donnemartin/system-design-primer#representational-state-transfer-rest)：
我们用的是公共的 [**REST API**](https://github.com/donnemartin/system-design-primer#representational-state-transfer-rest) ：
```
$ curl https://social.com/api/v1/friend_search?person_id=1234
@@ -243,13 +243,13 @@ $ curl https://social.com/api/v1/friend_search?person_id=1234
},
```
内部通信使用 [远端过程调用](https://github.com/donnemartin/system-design-primer#remote-procedure-call-rpc)。
内部通信使用 [远端过程调用](https://github.com/donnemartin/system-design-primer#remote-procedure-call-rpc) 。
## 第 4 步：扩展设计
> 在给定约束条件下，定义和确认瓶颈。
![Imgur](http://i.imgur.com/cdCv5g7.png)
![Imgur](http://i.imgur.com/cdCv5g7.png)
**重要：别简化从最初设计到最终设计的过程！**
@@ -261,14 +261,14 @@ $ curl https://social.com/api/v1/friend_search?person_id=1234
**避免重复讨论**，以下网址链接到 [系统设计主题](https://github.com/donnemartin/system-design-primer#index-of-system-design-topics) 相关的主流方案、折中方案和替代方案。
* [DNS](https://github.com/donnemartin/system-design-primer#domain-name-system)
* [负载均衡](https://github.com/donnemartin/system-design-primer#load-balancer)
* [横向扩展](https://github.com/donnemartin/system-design-primer#horizontal-scaling)
* [Web 服务器（反向代理）](https://github.com/donnemartin/system-design-primer#reverse-proxy-web-server)
* [API 服务器（应用层）](https://github.com/donnemartin/system-design-primer#application-layer)
* [缓存](https://github.com/donnemartin/system-design-primer#cache)
* [一致性模式](https://github.com/donnemartin/system-design-primer#consistency-patterns)
* [可用性模式](https://github.com/donnemartin/system-design-primer#availability-patterns)
* [DNS](https://github.com/donnemartin/system-design-primer#domain-name-system)
* [负载均衡](https://github.com/donnemartin/system-design-primer#load-balancer)
* [横向扩展](https://github.com/donnemartin/system-design-primer#horizontal-scaling)
* [Web 服务器（反向代理）](https://github.com/donnemartin/system-design-primer#reverse-proxy-web-server)
* [API 服务器（应用层）](https://github.com/donnemartin/system-design-primer#application-layer)
* [缓存](https://github.com/donnemartin/system-design-primer#cache)
* [一致性模式](https://github.com/donnemartin/system-design-primer#consistency-patterns)
* [可用性模式](https://github.com/donnemartin/system-design-primer#availability-patterns)
解决 **平均** 每秒 400 次请求的限制（峰值），人员数据可以存在例如 Redis 或 Memcached 这样的 **内存** 中以减少响应次数和下游流量通信服务。这尤其在用户执行多次连续查询和查询哪些广泛连接的人时十分有用。从内存中读取 1MB 数据大约要 250 微秒，从 SSD 中读取同样大小的数据时间要长 4 倍，从硬盘要长 80 倍。<sup><a href=https://github.com/donnemartin/system-design-primer#latency-numbers-every-programmer-should-know>1</a></sup>
@@ -279,9 +279,9 @@ $ curl https://social.com/api/v1/friend_search?person_id=1234
* 在同一台 **人员服务器** 上托管批处理同一批朋友查找减少机器跳转
* 通过地理位置 [拆分](https://github.com/donnemartin/system-design-primer#sharding) **人员服务器** 来进一步优化，因为朋友通常住得都比较近
* 同时进行两个 BFS 查找，一个从 source 开始，一个从 destination 开始，然后合并两个路径
* 从有庞大朋友圈的人开始找起，这样更有可能减小当前用户和搜索目标之间的 [离散度数](https://en.wikipedia.org/wiki/Six_degrees_of_separation)
* 从有庞大朋友圈的人开始找起，这样更有可能减小当前用户和搜索目标之间的 [离散度数](https://en.wikipedia.org/wiki/Six_degrees_of_separation)
* 在询问用户是否继续查询之前设置基于时间或跳跃数阈值，当在某些案例中搜索耗费时间过长时。
* 使用类似 [Neo4j](https://neo4j.com/) 的 **图数据库** 或图特定查询语法，例如 [GraphQL](http://graphql.org/)（如果没有禁止使用 **图数据库** 的限制的话）
* 使用类似 [Neo4j](https://neo4j.com/) 的 **图数据库** 或图特定查询语法，例如 [GraphQL](http://graphql.org/) （如果没有禁止使用 **图数据库** 的限制的话）
## 额外的话题
@@ -289,58 +289,58 @@ $ curl https://social.com/api/v1/friend_search?person_id=1234
### SQL 扩展模式
* [读取副本](https://github.com/donnemartin/system-design-primer#master-slave-replication)
* [集合](https://github.com/donnemartin/system-design-primer#federation)
* [分区](https://github.com/donnemartin/system-design-primer#sharding)
* [反规范化](https://github.com/donnemartin/system-design-primer#denormalization)
* [SQL 调优](https://github.com/donnemartin/system-design-primer#sql-tuning)
* [读取副本](https://github.com/donnemartin/system-design-primer#master-slave-replication)
* [集合](https://github.com/donnemartin/system-design-primer#federation)
* [分区](https://github.com/donnemartin/system-design-primer#sharding)
* [反规范化](https://github.com/donnemartin/system-design-primer#denormalization)
* [SQL 调优](https://github.com/donnemartin/system-design-primer#sql-tuning)
#### NoSQL
* [键值存储](https://github.com/donnemartin/system-design-primer#key-value-store)
* [文档存储](https://github.com/donnemartin/system-design-primer#document-store)
* [宽表存储](https://github.com/donnemartin/system-design-primer#wide-column-store)
* [图数据库](https://github.com/donnemartin/system-design-primer#graph-database)
* [SQL vs NoSQL](https://github.com/donnemartin/system-design-primer#sql-or-nosql)
* [键值存储](https://github.com/donnemartin/system-design-primer#key-value-store)
* [文档存储](https://github.com/donnemartin/system-design-primer#document-store)
* [宽表存储](https://github.com/donnemartin/system-design-primer#wide-column-store)
* [图数据库](https://github.com/donnemartin/system-design-primer#graph-database)
* [SQL vs NoSQL](https://github.com/donnemartin/system-design-primer#sql-or-nosql)
### 缓存
* 缓存到哪里
* [客户端缓存](https://github.com/donnemartin/system-design-primer#client-caching)
* [CDN 缓存](https://github.com/donnemartin/system-design-primer#cdn-caching)
* [Web 服务缓存](https://github.com/donnemartin/system-design-primer#web-server-caching)
* [数据库缓存](https://github.com/donnemartin/system-design-primer#database-caching)
* [应用缓存](https://github.com/donnemartin/system-design-primer#application-caching)
* [客户端缓存](https://github.com/donnemartin/system-design-primer#client-caching)
* [CDN 缓存](https://github.com/donnemartin/system-design-primer#cdn-caching)
* [Web 服务缓存](https://github.com/donnemartin/system-design-primer#web-server-caching)
* [数据库缓存](https://github.com/donnemartin/system-design-primer#database-caching)
* [应用缓存](https://github.com/donnemartin/system-design-primer#application-caching)
* 缓存什么
* [数据库请求层缓存](https://github.com/donnemartin/system-design-primer#caching-at-the-database-query-level)
* [对象层缓存](https://github.com/donnemartin/system-design-primer#caching-at-the-object-level)
* [数据库请求层缓存](https://github.com/donnemartin/system-design-primer#caching-at-the-database-query-level)
* [对象层缓存](https://github.com/donnemartin/system-design-primer#caching-at-the-object-level)
* 何时更新缓存
* [预留缓存](https://github.com/donnemartin/system-design-primer#cache-aside)
* [完全写入](https://github.com/donnemartin/system-design-primer#write-through)
* [延迟写 (写回)](https://github.com/donnemartin/system-design-primer#write-behind-write-back)
* [事先更新](https://github.com/donnemartin/system-design-primer#refresh-ahead)
* [预留缓存](https://github.com/donnemartin/system-design-primer#cache-aside)
* [完全写入](https://github.com/donnemartin/system-design-primer#write-through)
* [延迟写 (写回) ](https://github.com/donnemartin/system-design-primer#write-behind-write-back)
* [事先更新](https://github.com/donnemartin/system-design-primer#refresh-ahead)
### 异步性和微服务
* [消息队列](https://github.com/donnemartin/system-design-primer#message-queues)
* [任务队列](https://github.com/donnemartin/system-design-primer#task-queues)
* [回退压力](https://github.com/donnemartin/system-design-primer#back-pressure)
* [微服务](https://github.com/donnemartin/system-design-primer#microservices)
* [消息队列](https://github.com/donnemartin/system-design-primer#message-queues)
* [任务队列](https://github.com/donnemartin/system-design-primer#task-queues)
* [回退压力](https://github.com/donnemartin/system-design-primer#back-pressure)
* [微服务](https://github.com/donnemartin/system-design-primer#microservices)
### 沟通
* 关于折中方案的讨论:
* 客户端的外部通讯 - [遵循 REST 的 HTTP APIs](https://github.com/donnemartin/system-design-primer#representational-state-transfer-rest)
* 内部通讯 - [RPC](https://github.com/donnemartin/system-design-primer#remote-procedure-call-rpc)
* [服务探索](https://github.com/donnemartin/system-design-primer#service-discovery)
* 客户端的外部通讯 - [遵循 REST 的 HTTP APIs](https://github.com/donnemartin/system-design-primer#representational-state-transfer-rest)
* 内部通讯 - [RPC](https://github.com/donnemartin/system-design-primer#remote-procedure-call-rpc)
* [服务探索](https://github.com/donnemartin/system-design-primer#service-discovery)
### 安全性
参考 [安全章节](https://github.com/donnemartin/system-design-primer#security)
参考 [安全章节](https://github.com/donnemartin/system-design-primer#security)
### 延迟数字指标
查阅 [每个程序员必懂的延迟数字](https://github.com/donnemartin/system-design-primer#latency-numbers-every-programmer-should-know)
查阅 [每个程序员必懂的延迟数字](https://github.com/donnemartin/system-design-primer#latency-numbers-every-programmer-should-know)
### 正在进行

176
solutions/system_design/social_graph/README.md
View File

@@ -29,7 +29,7 @@ Without an interviewer to address clarifying questions, we'll define some use ca
* 50 friends per user average
* 1 billion friend searches per month
Exercise the use of more traditional systems - don't use graph-specific solutions such as [GraphQL](http://graphql.org/) or a graph database like [Neo4j](https://neo4j.com/)
Exercise the use of more traditional systems - don't use graph-specific solutions such as [GraphQL](http://graphql.org/) or a graph database like [Neo4j](https://neo4j.com/)
#### Calculate usage
@@ -50,7 +50,7 @@ Handy conversion guide:
> Outline a high level design with all important components.
![Imgur](http://i.imgur.com/wxXyq2J.png)
![Imgur](http://i.imgur.com/wxXyq2J.png)
## Step 3: Design core components
@@ -60,40 +60,40 @@ Handy conversion guide:
**Clarify with your interviewer how much code you are expected to write**.
Without the constraint of millions of users (vertices) and billions of friend relationships (edges), we could solve this unweighted shortest path task with a general BFS approach:
Without the constraint of millions of users (vertices) and billions of friend relationships (edges) , we could solve this unweighted shortest path task with a general BFS approach:
```python
class Graph(Graph):
class Graph(Graph) :
def shortest_path(self, source, dest):
def shortest_path(self, source, dest) :
if source is None or dest is None:
return None
if source is dest:
return [source.key]
prev_node_keys = self._shortest_path(source, dest)
prev_node_keys = self._shortest_path(source, dest)
if prev_node_keys is None:
return None
else:
path_ids = [dest.key]
prev_node_key = prev_node_keys[dest.key]
while prev_node_key is not None:
path_ids.append(prev_node_key)
path_ids.append(prev_node_key)
prev_node_key = prev_node_keys[prev_node_key]
return path_ids[::-1]
def _shortest_path(self, source, dest):
queue = deque()
queue.append(source)
def _shortest_path(self, source, dest) :
queue = deque()
queue.append(source)
prev_node_keys = {source.key: None}
source.visit_state = State.visited
while queue:
node = queue.popleft()
node = queue.popleft()
if node is dest:
return prev_node_keys
prev_node = node
for adj_node in node.adj_nodes.values():
for adj_node in node.adj_nodes.values() :
if adj_node.visit_state == State.unvisited:
queue.append(adj_node)
queue.append(adj_node)
prev_node_keys[adj_node.key] = prev_node.key
adj_node.visit_state = State.visited
return None
@@ -101,7 +101,7 @@ class Graph(Graph):
We won't be able to fit all users on the same machine, we'll need to [shard](https://github.com/donnemartin/system-design-primer#sharding) users across **Person Servers** and access them with a **Lookup Service**.
* The **Client** sends a request to the **Web Server**, running as a [reverse proxy](https://github.com/donnemartin/system-design-primer#reverse-proxy-web-server)
* The **Client** sends a request to the **Web Server**, running as a [reverse proxy](https://github.com/donnemartin/system-design-primer#reverse-proxy-web-server)
* The **Web Server** forwards the request to the **Search API** server
* The **Search API** server forwards the request to the **User Graph Service**
* The **User Graph Service** does the following:
@@ -109,7 +109,7 @@ We won't be able to fit all users on the same machine, we'll need to [shard](htt
* Finds the appropriate **Person Server** to retrieve the current user's list of `friend_ids`
* Runs a BFS search using the current user as the `source` and the current user's `friend_ids` as the ids for each `adjacent_node`
* To get the `adjacent_node` from a given id:
* The **User Graph Service** will *again* need to communicate with the **Lookup Service** to determine which **Person Server** stores the`adjacent_node` matching the given id (potential for optimization)
* The **User Graph Service** will *again* need to communicate with the **Lookup Service** to determine which **Person Server** stores the`adjacent_node` matching the given id (potential for optimization)
**Clarify with your interviewer how much code you should be writing**.
@@ -118,43 +118,43 @@ We won't be able to fit all users on the same machine, we'll need to [shard](htt
**Lookup Service** implementation:
```python
class LookupService(object):
class LookupService(object) :
def __init__(self):
self.lookup = self._init_lookup() # key: person_id, value: person_server
def __init__(self) :
self.lookup = self._init_lookup() # key: person_id, value: person_server
def _init_lookup(self):
def _init_lookup(self) :
...
def lookup_person_server(self, person_id):
def lookup_person_server(self, person_id) :
return self.lookup[person_id]
```
**Person Server** implementation:
```python
class PersonServer(object):
class PersonServer(object) :
def __init__(self):
def __init__(self) :
self.people = {} # key: person_id, value: person
def add_person(self, person):
def add_person(self, person) :
...
def people(self, ids):
def people(self, ids) :
results = []
for id in ids:
if id in self.people:
results.append(self.people[id])
results.append(self.people[id])
return results
```
**Person** implementation:
```python
class Person(object):
class Person(object) :
def __init__(self, id, name, friend_ids):
def __init__(self, id, name, friend_ids) :
self.id = id
self.name = name
self.friend_ids = friend_ids
@@ -163,21 +163,21 @@ class Person(object):
**User Graph Service** implementation:
```python
class UserGraphService(object):
class UserGraphService(object) :
def __init__(self, lookup_service):
def __init__(self, lookup_service) :
self.lookup_service = lookup_service
def person(self, person_id):
person_server = self.lookup_service.lookup_person_server(person_id)
return person_server.people([person_id])
def person(self, person_id) :
person_server = self.lookup_service.lookup_person_server(person_id)
return person_server.people([person_id])
def shortest_path(self, source_key, dest_key):
def shortest_path(self, source_key, dest_key) :
if source_key is None or dest_key is None:
return None
if source_key is dest_key:
return [source_key]
prev_node_keys = self._shortest_path(source_key, dest_key)
prev_node_keys = self._shortest_path(source_key, dest_key)
if prev_node_keys is None:
return None
else:
@@ -185,40 +185,40 @@ class UserGraphService(object):
path_ids = [dest_key]
prev_node_key = prev_node_keys[dest_key]
while prev_node_key is not None:
path_ids.append(prev_node_key)
path_ids.append(prev_node_key)
prev_node_key = prev_node_keys[prev_node_key]
# Reverse the list since we iterated backwards
return path_ids[::-1]
def _shortest_path(self, source_key, dest_key, path):
def _shortest_path(self, source_key, dest_key, path) :
# Use the id to get the Person
source = self.person(source_key)
source = self.person(source_key)
# Update our bfs queue
queue = deque()
queue.append(source)
queue = deque()
queue.append(source)
# prev_node_keys keeps track of each hop from
# the source_key to the dest_key
prev_node_keys = {source_key: None}
# We'll use visited_ids to keep track of which nodes we've
# visited, which can be different from a typical bfs where
# this can be stored in the node itself
visited_ids = set()
visited_ids.add(source.id)
visited_ids = set()
visited_ids.add(source.id)
while queue:
node = queue.popleft()
node = queue.popleft()
if node.key is dest_key:
return prev_node_keys
prev_node = node
for friend_id in node.friend_ids:
if friend_id not in visited_ids:
friend_node = self.person(friend_id)
queue.append(friend_node)
friend_node = self.person(friend_id)
queue.append(friend_node)
prev_node_keys[friend_id] = prev_node.key
visited_ids.add(friend_id)
visited_ids.add(friend_id)
return None
```
We'll use a public [**REST API**](https://github.com/donnemartin/system-design-primer#representational-state-transfer-rest):
We'll use a public [**REST API**](https://github.com/donnemartin/system-design-primer#representational-state-transfer-rest) :
```
$ curl https://social.com/api/v1/friend_search?person_id=1234
@@ -244,13 +244,13 @@ Response:
},
```
For internal communications, we could use [Remote Procedure Calls](https://github.com/donnemartin/system-design-primer#remote-procedure-call-rpc).
For internal communications, we could use [Remote Procedure Calls](https://github.com/donnemartin/system-design-primer#remote-procedure-call-rpc) .
## Step 4: Scale the design
> Identify and address bottlenecks, given the constraints.
![Imgur](http://i.imgur.com/cdCv5g7.png)
![Imgur](http://i.imgur.com/cdCv5g7.png)
**Important: Do not simply jump right into the final design from the initial design!**
@@ -262,16 +262,16 @@ We'll introduce some components to complete the design and to address scalabilit
*To avoid repeating discussions*, refer to the following [system design topics](https://github.com/donnemartin/system-design-primer#index-of-system-design-topics) for main talking points, tradeoffs, and alternatives:
* [DNS](https://github.com/donnemartin/system-design-primer#domain-name-system)
* [Load balancer](https://github.com/donnemartin/system-design-primer#load-balancer)
* [Horizontal scaling](https://github.com/donnemartin/system-design-primer#horizontal-scaling)
* [Web server (reverse proxy)](https://github.com/donnemartin/system-design-primer#reverse-proxy-web-server)
* [API server (application layer)](https://github.com/donnemartin/system-design-primer#application-layer)
* [Cache](https://github.com/donnemartin/system-design-primer#cache)
* [Consistency patterns](https://github.com/donnemartin/system-design-primer#consistency-patterns)
* [Availability patterns](https://github.com/donnemartin/system-design-primer#availability-patterns)
* [DNS](https://github.com/donnemartin/system-design-primer#domain-name-system)
* [Load balancer](https://github.com/donnemartin/system-design-primer#load-balancer)
* [Horizontal scaling](https://github.com/donnemartin/system-design-primer#horizontal-scaling)
* [Web server (reverse proxy) ](https://github.com/donnemartin/system-design-primer#reverse-proxy-web-server)
* [API server (application layer) ](https://github.com/donnemartin/system-design-primer#application-layer)
* [Cache](https://github.com/donnemartin/system-design-primer#cache)
* [Consistency patterns](https://github.com/donnemartin/system-design-primer#consistency-patterns)
* [Availability patterns](https://github.com/donnemartin/system-design-primer#availability-patterns)
To address the constraint of 400 *average* read requests per second (higher at peak), person data can be served from a **Memory Cache** such as Redis or Memcached to reduce response times and to reduce traffic to downstream services. This could be especially useful for people who do multiple searches in succession and for people who are well-connected. Reading 1 MB sequentially from memory takes about 250 microseconds, while reading from SSD takes 4x and from disk takes 80x longer.<sup><a href=https://github.com/donnemartin/system-design-primer#latency-numbers-every-programmer-should-know>1</a></sup>
To address the constraint of 400 *average* read requests per second (higher at peak) , person data can be served from a **Memory Cache** such as Redis or Memcached to reduce response times and to reduce traffic to downstream services. This could be especially useful for people who do multiple searches in succession and for people who are well-connected. Reading 1 MB sequentially from memory takes about 250 microseconds, while reading from SSD takes 4x and from disk takes 80x longer.<sup><a href=https://github.com/donnemartin/system-design-primer#latency-numbers-every-programmer-should-know>1</a></sup>
Below are further optimizations:
@@ -282,7 +282,7 @@ Below are further optimizations:
* Do two BFS searches at the same time, one starting from the source, and one from the destination, then merge the two paths
* Start the BFS search from people with large numbers of friends, as they are more likely to reduce the number of [degrees of separation](https://en.wikipedia.org/wiki/Six_degrees_of_separation) between the current user and the search target
* Set a limit based on time or number of hops before asking the user if they want to continue searching, as searching could take a considerable amount of time in some cases
* Use a **Graph Database** such as [Neo4j](https://neo4j.com/) or a graph-specific query language such as [GraphQL](http://graphql.org/) (if there were no constraint preventing the use of **Graph Databases**)
* Use a **Graph Database** such as [Neo4j](https://neo4j.com/) or a graph-specific query language such as [GraphQL](http://graphql.org/) (if there were no constraint preventing the use of **Graph Databases**)
## Additional talking points
@@ -290,58 +290,58 @@ Below are further optimizations:
### SQL scaling patterns
* [Read replicas](https://github.com/donnemartin/system-design-primer#master-slave-replication)
* [Federation](https://github.com/donnemartin/system-design-primer#federation)
* [Sharding](https://github.com/donnemartin/system-design-primer#sharding)
* [Denormalization](https://github.com/donnemartin/system-design-primer#denormalization)
* [SQL Tuning](https://github.com/donnemartin/system-design-primer#sql-tuning)
* [Read replicas](https://github.com/donnemartin/system-design-primer#master-slave-replication)
* [Federation](https://github.com/donnemartin/system-design-primer#federation)
* [Sharding](https://github.com/donnemartin/system-design-primer#sharding)
* [Denormalization](https://github.com/donnemartin/system-design-primer#denormalization)
* [SQL Tuning](https://github.com/donnemartin/system-design-primer#sql-tuning)
#### NoSQL
* [Key-value store](https://github.com/donnemartin/system-design-primer#key-value-store)
* [Document store](https://github.com/donnemartin/system-design-primer#document-store)
* [Wide column store](https://github.com/donnemartin/system-design-primer#wide-column-store)
* [Graph database](https://github.com/donnemartin/system-design-primer#graph-database)
* [SQL vs NoSQL](https://github.com/donnemartin/system-design-primer#sql-or-nosql)
* [Key-value store](https://github.com/donnemartin/system-design-primer#key-value-store)
* [Document store](https://github.com/donnemartin/system-design-primer#document-store)
* [Wide column store](https://github.com/donnemartin/system-design-primer#wide-column-store)
* [Graph database](https://github.com/donnemartin/system-design-primer#graph-database)
* [SQL vs NoSQL](https://github.com/donnemartin/system-design-primer#sql-or-nosql)
### Caching
* Where to cache
* [Client caching](https://github.com/donnemartin/system-design-primer#client-caching)
* [CDN caching](https://github.com/donnemartin/system-design-primer#cdn-caching)
* [Web server caching](https://github.com/donnemartin/system-design-primer#web-server-caching)
* [Database caching](https://github.com/donnemartin/system-design-primer#database-caching)
* [Application caching](https://github.com/donnemartin/system-design-primer#application-caching)
* [Client caching](https://github.com/donnemartin/system-design-primer#client-caching)
* [CDN caching](https://github.com/donnemartin/system-design-primer#cdn-caching)
* [Web server caching](https://github.com/donnemartin/system-design-primer#web-server-caching)
* [Database caching](https://github.com/donnemartin/system-design-primer#database-caching)
* [Application caching](https://github.com/donnemartin/system-design-primer#application-caching)
* What to cache
* [Caching at the database query level](https://github.com/donnemartin/system-design-primer#caching-at-the-database-query-level)
* [Caching at the object level](https://github.com/donnemartin/system-design-primer#caching-at-the-object-level)
* [Caching at the database query level](https://github.com/donnemartin/system-design-primer#caching-at-the-database-query-level)
* [Caching at the object level](https://github.com/donnemartin/system-design-primer#caching-at-the-object-level)
* When to update the cache
* [Cache-aside](https://github.com/donnemartin/system-design-primer#cache-aside)
* [Write-through](https://github.com/donnemartin/system-design-primer#write-through)
* [Write-behind (write-back)](https://github.com/donnemartin/system-design-primer#write-behind-write-back)
* [Refresh ahead](https://github.com/donnemartin/system-design-primer#refresh-ahead)
* [Cache-aside](https://github.com/donnemartin/system-design-primer#cache-aside)
* [Write-through](https://github.com/donnemartin/system-design-primer#write-through)
* [Write-behind (write-back) ](https://github.com/donnemartin/system-design-primer#write-behind-write-back)
* [Refresh ahead](https://github.com/donnemartin/system-design-primer#refresh-ahead)
### Asynchronism and microservices
* [Message queues](https://github.com/donnemartin/system-design-primer#message-queues)
* [Task queues](https://github.com/donnemartin/system-design-primer#task-queues)
* [Back pressure](https://github.com/donnemartin/system-design-primer#back-pressure)
* [Microservices](https://github.com/donnemartin/system-design-primer#microservices)
* [Message queues](https://github.com/donnemartin/system-design-primer#message-queues)
* [Task queues](https://github.com/donnemartin/system-design-primer#task-queues)
* [Back pressure](https://github.com/donnemartin/system-design-primer#back-pressure)
* [Microservices](https://github.com/donnemartin/system-design-primer#microservices)
### Communications
* Discuss tradeoffs:
* External communication with clients - [HTTP APIs following REST](https://github.com/donnemartin/system-design-primer#representational-state-transfer-rest)
* Internal communications - [RPC](https://github.com/donnemartin/system-design-primer#remote-procedure-call-rpc)
* [Service discovery](https://github.com/donnemartin/system-design-primer#service-discovery)
* External communication with clients - [HTTP APIs following REST](https://github.com/donnemartin/system-design-primer#representational-state-transfer-rest)
* Internal communications - [RPC](https://github.com/donnemartin/system-design-primer#remote-procedure-call-rpc)
* [Service discovery](https://github.com/donnemartin/system-design-primer#service-discovery)
### Security
Refer to the [security section](https://github.com/donnemartin/system-design-primer#security).
Refer to the [security section](https://github.com/donnemartin/system-design-primer#security) .
### Latency numbers
See [Latency numbers every programmer should know](https://github.com/donnemartin/system-design-primer#latency-numbers-every-programmer-should-know).
See [Latency numbers every programmer should know](https://github.com/donnemartin/system-design-primer#latency-numbers-every-programmer-should-know) .
### Ongoing

44
solutions/system_design/social_graph/social_graph_snippets.py
View File

@@ -3,70 +3,70 @@ from collections import deque
from enum import Enum
class State(Enum):
class State(Enum) :
unvisited = 0
visited = 1
class Graph(object):
class Graph(object) :
def bfs(self, source, dest):
def bfs(self, source, dest) :
if source is None:
return False
queue = deque()
queue.append(source)
queue = deque()
queue.append(source)
source.visit_state = State.visited
while queue:
node = queue.popleft()
print(node)
node = queue.popleft()
print(node)
if dest is node:
return True
for adjacent_node in node.adj_nodes.values():
for adjacent_node in node.adj_nodes.values() :
if adjacent_node.visit_state == State.unvisited:
queue.append(adjacent_node)
queue.append(adjacent_node)
adjacent_node.visit_state = State.visited
return False
class Person(object):
class Person(object) :
def __init__(self, id, name):
def __init__(self, id, name) :
self.id = id
self.name = name
self.friend_ids = []
class LookupService(object):
class LookupService(object) :
def __init__(self):
def __init__(self) :
self.lookup = {} # key: person_id, value: person_server
def get_person(self, person_id):
def get_person(self, person_id) :
person_server = self.lookup[person_id]
return person_server.people[person_id]
class PersonServer(object):
class PersonServer(object) :
def __init__(self):
def __init__(self) :
self.people = {} # key: person_id, value: person
def get_people(self, ids):
def get_people(self, ids) :
results = []
for id in ids:
if id in self.people:
results.append(self.people[id])
results.append(self.people[id])
return results
class UserGraphService(object):
class UserGraphService(object) :
def __init__(self, person_ids, lookup):
def __init__(self, person_ids, lookup) :
self.lookup = lookup
self.person_ids = person_ids
self.visited_ids = set()
self.visited_ids = set()
def bfs(self, source, dest):
def bfs(self, source, dest) :
# Use self.visited_ids to track visited nodes
# Use self.lookup to translate a person_id to a Person
pass