千帆过尽，独具只眼 | 从1K+顶级Github机器学习项目中发现的秘密

new_profile.info()

import matplotlib.pyplot as plt
import numpy as np
import plotly.express as px # for plotting
import altair as alt # for plotting
import datapane as dp # for creating a report for your findings

top_followers = new_profile.sort_values(by='followers', axis=0, ascending=False)

fig = px.bar(top_followers, 
             x='user_name', 
             y='followers',
             hover_data=['followers'],
            )
fig.show()

>>> top_n = int(len(top_followers) * 0.01)
12
>>> sum(top_followers.iloc[0: top_n,:].loc[:, 'followers'])/sum(top_followers.followers)
0.41293075864408607

figs = [] # list to save all the plots and table

features = ['followers',
               'following',
               'total_stars',
               'max_star',
               'forks',
                'contribution']
for col in features:
    top_col = new_profile.sort_values(by=col, axis=0, ascending=False)
    
    log_y = False  
    #change scale of y-axis of every feature to log except contribution
    if col != 'contribution':
        log_y = True
        
    fig = px.bar(top_col,
             x='user_name', 
             y=col,
             hover_data=[col],
             log_y = log_y
            )
    
    fig.update_layout({'plot_bgcolor': 'rgba(36, 83, 97, 0.06)'}) #change background coor
    
    fig.show()
    
    figs.append(dp.Plot(fig))

correlation = px.scatter_matrix(new_profile, dimensions=['forks', 'total_stars', 'followers',
                                 'following', 'max_star','contribution'],
                               title='Correlation between datapoints',
                               width=800, height=800)
correlation.show()

corr = new_profile.corr()

figs.append(dp.Plot(correlation))
figs.append(dp.Table(corr))
corr

# Collect languages from all repos of al users
languages = []
for language in list(new_profile['languages']):
    try:
        languages += language
    except:
        languages += ['None'] 
        
# Count the frequency of each language
from collections import Counter 
occ = dict(Counter(languages))
 
# Remove languages below count of 10
top_languages = [(language, frequency) for language, frequency in occ.items() if frequency > 10]
top_languages = list(zip(*top_languages))

language_df = pd.DataFrame(data = {'languages': top_languages[0],
                           'frequency': top_languages[1]})

language_df.sort_values(by='frequency', axis=0, inplace=True, ascending=False)

language = px.bar(language_df, y='frequency', x='languages',
      title='Frequency of languages')

figs.append(dp.Plot(language))

language.show()

import altair as alt

hireable = alt.Chart(new_profile).transform_aggregate(
    count='count()',
    groupby=['hireable']
).mark_bar().encode(
    x='hireable:O',
    y='count:Q')

figs.append(dp.Plot(hireable))
hireable

from geopy.geocoders import Nominatim

locations = list(new_profile['location'])

# Extract lats and lons
lats = []
lons = []
exceptions = []

for loc in locations:
    try:
        location = geolocator.geocode(loc)
        lats.append(location.latitude)
        lons.append(location.longitude)
        print(location.address)
    except:
        print('exception', loc)
        exceptions.append(loc)
        
print(len(exceptions)) # output: 17

 # Remove the locations not found in map
location_df['latitude'] = lats
location_df['longitude'] = lons

# Visualize with Plotly's scatter_geo
m = px.scatter_geo(location_df, lat='latitude', lon='longitude',
                 color='total_stars', size='forks',
                 hover_data=['user_name','followers'],
                 title='Locations of Top Users')
m.show()

figs.append(dp.Plot(m))

import string
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.tokenize import word_tokenize
from wordcloud import WordCloud, STOPWORDS 
import matplotlib.pyplot as plt

nltk.download('stopwords')
nltk.download('punkt')
nltk.download('wordnet')

      
def process_text(features):
  '''Function to process texts'''
    
    features = [row for row in features if row != None]
    
    text = ' '.join(features)
    
    
    
    # lowercase
    text = text.lower()

    #remove punctuation
    text = text.translate(str.maketrans('', '', string.punctuation))

    #remove stopwords
    stop_words = set(stopwords.words('english'))

    #tokenize
    tokens = word_tokenize(text)
    new_text = [i for i in tokens if not i in stop_words]
    
    new_text = ' '.join(new_text)
    
    return new_text

def make_wordcloud(new_text):
  'funciton to make wordcloud'
    
    wordcloud = WordCloud(width = 800, height = 800, 
                background_color ='white', 
                min_font_size = 10).generate(new_text) 

    
    fig = plt.figure(figsize = (8, 8), facecolor = None) 
    plt.imshow(wordcloud) 
    plt.axis("off") 
    plt.tight_layout(pad = 0) 

    plt.show() 
    
    return fig
    
descriptions = []
for desc in new_profile['descriptions']:
    try:
        descriptions += desc
        
    except:
        pass

descriptions = process_text(bios)

cloud = make_wordcloud(text)

figs.append(dp.Plot(cloud))

bios = []
for bio in new_profile['bio']:
    try:
        bios.append(bio)
        
    except:
        pass
      
text = process_text(bios)

cloud = make_wordcloud(text)

figs.append(dp.Plot(cloud))