Дневники
Тесты
Сообщества
Альбомы
турция
5 лет назад от Алекс Кац
Импортируем библиотеки
In [1]:
import pandas as pdimport numpy as npimport seaborn as snsimport matplotlib.pyplot as pltplt.style.use("ggplot")import warningswarnings.filterwarnings("ignore")from sklearn.preprocessing import StandardScalerfrom sklearn.decomposition import PCAfrom sklearn import datasetsМодуль А. Предобработка больших данных
Загрузка данных из модуля 1.1
In [3]:
low_memory=Falsedata = pd.read_csv("real_estate_data.csv", skipinitialspace = True)In [4]:
data.head(6)Out[4]:
idtypesub_typestart_dateend_datelisting_typetombuilding_agetotal_floor_countfloor_noroom_countsizeaddressfurnishedheating_typepriceprice_currency
01KonutRezidans12/10/181/9/19230020 ve üzeri22+190.0İstanbul/Kartal/KordonboyuNaNFancoil3500.0TRY
12KonutDaire2/13/19NaN114020 ve üzeri20 ve üzeri1+043.0İstanbul/Kartal/KordonboyuNaNFancoil490000.0TRY
23KonutDaire10/9/1811/8/1813001Yüksek Giriş2+1NaNTekirdağ/Çorlu/ReşadiyeNaNFancoil155000.0TRY
34KonutRezidans9/10/1810/10/18130320 ve üzeri20 ve üzeri6+1450.0İstanbul/Beşiktaş/LeventNaNFancoil32500000.0TRY
45KonutRezidans12/10/181/9/19130020 ve üzeri22+190.0İstanbul/Kartal/KordonboyuNaNFancoil1450000.0TRY
56KonutRezidans11/9/1812/9/18130210-20 arası101+145.0İstanbul/Maltepe/AltayçeşmeNaNFancoil780000.0TRY
In [5]:
data.info()RangeIndex: 403487 entries, 0 to 403486Data columns (total 17 columns):id 403487 non-null int64type 403487 non-null objectsub_type 403487 non-null objectstart_date 403487 non-null objectend_date 266298 non-null objectlisting_type 403487 non-null int64tom 403487 non-null int64building_age 376097 non-null objecttotal_floor_count 375466 non-null objectfloor_no 368191 non-null objectroom_count 403487 non-null objectsize 257481 non-null float64address 403487 non-null objectfurnished 0 non-null float64heating_type 375517 non-null objectprice 402772 non-null float64price_currency 402772 non-null objectdtypes: float64(3), int64(3), object(11)memory usage: 52.3+ MB-id -furnished start end adr
In [6]:
data.shapeOut[6]:
(403487, 17)In [7]:
data.isna().sum()Out[7]:
id 0type 0sub_type 0start_date 0end_date 137189listing_type 0tom 0building_age 27390total_floor_count 28021floor_no 35296room_count 0size 146006address 0furnished 403487heating_type 27970price 715price_currency 715dtype: int64Очистим данные от технической информации (id, furnished, start_date, end_date, address)
In [8]:
data.drop("id", axis = 1, inplace=True)data.drop("furnished", axis = 1, inplace=True)data.drop("start_date", axis = 1, inplace=True)data.drop("end_date", axis = 1, inplace=True)data.drop("address", axis = 1, inplace=True)Мы удалили id, furnished, start_date, end_date, address
In [9]:
data.head(5)Out[9]:
typesub_typelisting_typetombuilding_agetotal_floor_countfloor_noroom_countsizeheating_typepriceprice_currency
0KonutRezidans230020 ve üzeri22+190.0Fancoil3500.0TRY
1KonutDaire114020 ve üzeri20 ve üzeri1+043.0Fancoil490000.0TRY
2KonutDaire13001Yüksek Giriş2+1NaNFancoil155000.0TRY
3KonutRezidans130320 ve üzeri20 ve üzeri6+1450.0Fancoil32500000.0TRY
4KonutRezidans130020 ve üzeri22+190.0Fancoil1450000.0TRY
In [10]:
data.shapeOut[10]:
(403487, 12)1.2 Предобработка данных
Проанализируем значения атрибутов датасета
In [11]:
for col in data.columns: print("------") print("Column: ", col) print("Counts: ", data[col].value_counts()) print("Unique: ", data[col].unique()) print("Null: ", data[col].isna().sum())------Column: typeCounts: Konut 403487Name: type, dtype: int64Unique: ['Konut']Null: 0------Column: sub_typeCounts: Daire 354549Villa 21324Müstakil Ev 9563Rezidans 7716Yazlık 5929Komple Bina 2607Prefabrik Ev 679Çiftlik Evi 528Köşk / Konak / Yalı 301Yalı Dairesi 187Kooperatif 70Loft 34Name: sub_type, dtype: int64Unique: ['Rezidans' 'Daire' 'Villa' 'Müstakil Ev' 'Kooperatif' 'Yazlık' 'Komple Bina' 'Prefabrik Ev' 'Köşk / Konak / Yalı' 'Çiftlik Evi' 'Yalı Dairesi' 'Loft']Null: 0------Column: listing_typeCounts: 1 2870092 1142363 2242Name: listing_type, dtype: int64Unique: [2 1 3]Null: 0------Column: tomCounts: 30 8221660 223250 1115190 986861 67381 4855120 450491 433362 42922 42557 40836 38945 381114 37624 37418 373021 36929 365513 364215 335712 325892 323563 32153 300411 292620 291422 288627 287519 287516 2862 ... 149 700159 687160 679144 671166 657162 641108 637167 637129 612174 597168 585175 565163 565170 530176 521115 521165 501173 496158 494169 468172 465177 464180 457136 422179 379143 313157 287164 210178 196171 158Name: tom, Length: 181, dtype: int64Unique: [ 30 14 54 92 11 0 13 158 94 97 154 141 52 61 118 60 136 53 114 34 142 7 140 117 127 47 80 42 78 161 8 64 159 1 31 48 28 138 120 4 93 91 124 63 23 123 6 170 130 25 46 59 73 2 99 9 69 21 134 44 121 71 76 79 139 151 126 90 51 55 156 100 152 129 3 96 98 5 62 119 12 147 89 50 128 24 106 111 88 173 122 83 174 19 17 68 40 15 29 67 160 110 75 37 70 131 43 145 66 49 150 132 72 39 167 65 153 133 74 101 16 56 103 175 10 22 33 144 149 86 168 77 108 38 146 166 45 32 104 87 177 148 26 27 84 20 107 81 82 18 180 172 155 165 113 41 95 57 58 36 35 169 85 143 112 163 102 116 137 125 105 162 109 157 115 179 176 171 135 164 178]Null: 0------Column: building_ageCounts: 0 1401746-10 arası 5049511-15 arası 3230916-20 arası 313331 203554 1903221-25 arası 184382 174663 156515 1358926-30 arası 1058131-35 arası 426836-40 arası 134740 ve üzeri 1059Name: building_age, dtype: int64Unique: ['0' '3' '2' nan '1' '6-10 arası' '4' '16-20 arası' '26-30 arası' '40 ve üzeri' '11-15 arası' '21-25 arası' '5' '31-35 arası' '36-40 arası']Null: 27390------Column: total_floor_countCounts: 4 830823 779565 7010410-20 arası 365122 277426 2334810 125587 122848 112079 902920 ve üzeri 66791 4965Name: total_floor_count, dtype: int64Unique: ['20 ve üzeri' '1' '10-20 arası' '4' '2' '3' '8' '7' '6' '5' '10' nan '9']Null: 28021------Column: floor_noCounts: 2 609143 322001 28569Yüksek Giriş 24045Müstakil 211654 210493 20490Bahçe katı 190651 181874 134165 12495Giriş Katı 104315 86986 5116Kot 1 5036Kot 2 49872 49509 48556 46318 46087 439810 3863Kot 3 3793Çatı Katı 3566Zemin Kat 34417 3300Komple 295811 289412 2308Kot 4 226913 170220 ve üzeri 15638 149114 132815 911En Üst Kat 894Bodrum Kat 81516 60017 37318 334Teras Kat 29319 177Asma Kat 1212 1Name: floor_no, dtype: int64Unique: ['2' '20 ve üzeri' 'Yüksek Giriş' '10' '14' nan 'Kot 2' 'Asma Kat' 'Bahçe katı' '11' '3' '13' '7' '16' 'Müstakil' 'Zemin Kat' '19' '4' '5' 'En Üst Kat' '8' '15' '1' 'Giriş Katı' '9' 'Çatı Katı' '12' '17' '6' 'Kot 4' 'Kot 1' 'Kot 3' '18' 'Teras Kat' 'Komple' 'Bodrum Kat' 1 2 3 12 4 5 6 7 8]Null: 35296------Column: room_countCounts: 3+1 1573632+1 1386771+1 391344+1 374725+1 82194+2 45405+2 2926+ 28983+2 26731+0 26126+1 20016+2 15172+2 8367+1 5237+2 42810+0 3138+1 2188+2 1896+3 1675+3 1394+3 1337+3 869+1 719+3 659+2 618+3 558+4 3810+1 3510+2 309+5 279+4 2610+3 910+5 211+3 115+5 110+4 10+0 1Name: room_count, dtype: int64Unique: ['2+1' '1+0' '6+1' '1+1' '3+1' '4+1' '2+2' '9+2' '5+2' '6+3' '6+2' '4+3' '9+5' '5+1' '3+2' '+' '4+2' '7+2' '9+4' '8+2' '8+3' '10+0' '8+4' '8+1' '7+1' '10+1' '9+3' '7+3' '10+2' '5+3' '9+1' '0+0' '10+5' '10+3' '10+4' '11+3' '15+5']Null: 0------Column: sizeCounts: 90.0 16803100.0 16234120.0 15543110.0 1354680.0 11094130.0 1091885.0 10437140.0 9103150.0 858095.0 8036125.0 762375.0 6966135.0 651470.0 6264115.0 6244160.0 5923105.0 487360.0 4528180.0 4334145.0 424755.0 4134200.0 413450.0 397065.0 3751170.0 3499165.0 2631155.0 232845.0 2169190.0 1922175.0 1826 ... 1680.0 1359.0 1737990.0 1820.0 1909039.0 1586.0 11365.0 1446.0 129000.0 1905.0 1758.0 1677.0 1339.0 1679.0 1231280.0 13300.0 11320.0 1175000.0 1889.0 14500.0 13599.0 143818.0 11799.0 187851.0 1726.0 11736.0 1447.0 1891.0 11385.0 142262.0 1Name: size, Length: 943, dtype: int64Unique: [9.00000e+01 4.30000e+01 nan 4.50000e+02 4.50000e+01 1.60000e+02 1.40000e+02 5.50000e+02 1.25000e+02 1.65000e+02 2.00000e+02 8.50000e+01 8.40000e+01 5.00000e+01 5.50000e+01 1.00000e+02 7.00000e+01 3.00000e+01 6.18173e+05 2.25000e+02 1.58000e+02 5.20000e+01 3.87000e+02 1.05000e+02 4.00000e+02 6.10000e+01 1.00000e+00 2.15000e+02 4.70000e+01 1.50000e+02 1.90000e+02 1.20000e+02 6.00000e+01 1.85000e+02 2.27000e+02 6.80000e+01 4.80000e+01 2.50000e+02 1.80000e+02 2.45000e+02 4.00000e+01 1.73000e+02 3.20000e+02 6.50000e+01 2.30000e+02 2.26990e+04 2.22000e+02 1.30000e+02 2.80000e+01 1.12000e+02 3.20000e+01 4.20000e+01 3.30000e+01 1.55000e+02 1.70000e+02 2.55000e+02 8.00000e+01 1.36000e+02 1.93000e+02 3.67331e+05 8.60000e+01 1.57000e+02 1.35000e+02 1.04000e+02 1.45000e+02 2.70000e+02 1.10000e+02 3.50000e+01 2.85000e+02 5.80000e+01 3.00000e+02 6.30000e+01 4.32000e+02 2.20000e+02 3.30000e+02 2.60000e+02 5.00000e+02 5.60000e+01 9.70000e+01 3.90000e+02 2.30000e+01 3.63679e+05 1.89000e+02 7.50000e+01 8.70000e+01 1.34000e+02 1.68000e+02 1.61000e+02 1.42000e+02 3.80000e+01 1.11000e+02 2.00000e+00 2.40000e+02 8.73000e+02 1.75000e+02 6.70000e+01 6.08860e+05 2.72000e+02 5.40000e+01 8.11910e+04 1.95148e+05 6.20000e+01 2.23000e+02 3.60000e+02 1.15000e+02 9.30000e+01 8.60200e+03 2.21000e+02 9.05000e+02 2.00000e+03 9.50000e+01 2.32000e+02 1.08000e+02 1.22000e+02 7.60000e+01 1.00000e+03 1.17000e+02 9.90000e+01 1.72000e+03 1.62000e+02 6.90000e+02 1.02000e+02 1.35000e+03 6.40000e+01 1.26000e+02 9.49000e+02 1.84000e+02 7.70000e+01 8.80000e+01 3.56000e+02 2.10000e+01 9.60000e+01 1.95000e+02 4.20000e+02 1.14400e+03 1.96000e+02 7.00000e+03 3.00000e+00 3.13000e+02 1.51000e+02 3.50000e+02 2.05000e+02 1.18000e+02 5.10000e+01 7.50000e+02 2.50000e+03 1.24000e+02 4.35000e+02 1.06000e+02 4.90000e+01 4.60000e+01 2.46000e+02 6.00000e+02 2.10000e+02 2.63000e+02 9.80000e+01 2.50000e+01 1.66200e+03 1.39000e+02 1.29000e+02 9.98000e+03 1.49000e+02 1.69000e+04 1.48000e+02 7.80000e+01 1.80000e+03 3.90000e+01 2.59000e+02 1.23000e+02 1.31000e+02 2.89000e+02 6.90000e+01 4.05000e+02 7.90000e+01 4.80000e+02 1.37000e+02 1.47000e+02 2.28000e+02 3.74000e+02 3.10000e+02 1.33000e+02 7.30000e+01 2.49000e+02 4.36000e+02 8.90000e+01 1.00000e+01 3.39000e+02 7.20000e+01 2.35000e+02 2.99000e+02 1.43000e+02 2.48000e+02 1.19000e+02 1.87000e+02 8.20000e+01 2.74000e+02 3.80000e+02 8.49700e+03 2.95000e+02 3.95000e+02 4.00000e+03 1.28000e+02 9.85000e+02 3.69000e+02 1.79900e+03 1.19000e+03 3.70000e+02 3.23000e+02 3.48000e+02 1.78000e+02 7.40000e+01 4.54000e+02 4.40000e+01 7.49000e+02 9.20000e+01 1.32000e+02 3.19000e+02 5.30000e+01 1.46000e+02 6.41000e+02 1.41000e+02 1.14000e+02 1.63000e+03 1.21000e+02 1.38000e+02 3.06000e+02 1.76000e+02 9.10000e+01 4.44000e+02 1.79000e+02 2.12000e+02 9.00000e+02 1.03000e+02 1.52000e+02 5.90000e+01 3.70000e+01 8.10000e+01 2.40000e+01 1.16000e+02 4.38180e+04 1.27000e+02 1.64000e+02 7.78000e+02 8.30000e+01 1.56000e+02 2.00000e+01 3.15000e+02 1.80000e+01 6.50000e+02 9.40000e+01 2.26000e+02 3.40000e+02 3.35000e+02 1.59000e+02 3.40000e+01 2.01000e+02 7.00000e+02 4.73000e+02 2.37000e+02 1.03000e+03 1.13000e+02 3.65000e+02 1.69000e+02 1.77000e+02 1.63000e+02 2.65000e+02 1.09000e+02 6.40000e+02 1.00000e+04 9.50000e+02 8.00000e+00 4.22620e+04 6.60000e+01 5.00000e+00 2.90000e+02 6.01000e+02 6.20000e+02 3.00000e+03 7.60000e+02 1.90000e+03 1.98000e+02 7.02000e+02 1.54000e+02 4.26000e+02 2.80000e+02 1.74000e+02 5.67000e+02 2.67000e+02 3.95000e+03 5.08000e+02 8.49000e+02 1.53000e+02 1.44000e+02 6.60000e+02 2.14000e+02 1.10000e+03 5.66000e+02 2.08000e+02 2.07000e+02 5.70000e+01 3.62000e+02 4.25000e+02 2.34000e+02 2.24000e+02 2.75000e+02 1.99000e+02 1.07000e+02 2.57000e+02 3.12070e+04 1.88000e+02 3.83000e+02 4.49000e+02 1.20000e+01 5.83000e+02 2.03000e+02 2.17000e+02 5.20000e+02 1.83000e+02 1.01000e+02 2.98000e+02 5.60000e+02 1.91000e+02 1.72000e+02 1.67000e+02 3.71000e+02 4.23000e+02 1.30000e+01 1.92000e+02 4.48000e+02 7.10000e+01 1.86000e+02 4.99000e+02 1.97000e+02 2.16000e+02 9.44330e+04 4.10000e+02 3.34000e+02 7.00000e+00 1.71000e+02 1.11472e+05 7.05000e+02 1.82000e+02 8.65799e+05 3.49000e+02 4.06000e+02 1.50000e+03 6.85000e+02 3.85000e+03 4.52189e+05 9.00000e+03 3.45000e+02 4.60000e+02 6.30000e+02 7.68000e+02 2.47000e+02 6.77000e+02 1.50000e+01 1.46117e+05 9.92000e+02 2.64000e+02 1.19400e+03 1.66000e+02 2.29000e+02 4.70000e+02 3.47000e+02 2.20000e+01 1.07319e+05 1.20000e+03 7.01000e+02 2.86000e+02 7.89286e+05 2.19000e+02 3.25000e+02 5.40000e+02 2.70530e+04 2.04835e+05 8.25000e+02 2.80000e+03 2.04000e+02 1.05000e+03 2.18000e+02 4.15000e+02 3.05000e+02 4.30000e+02 5.25000e+02 3.60000e+01 2.10000e+03 3.36000e+02 9.99000e+02 1.81000e+02 3.75000e+02 2.55000e+03 8.70000e+02 3.28000e+02 5.47903e+05 7.40000e+02 4.10000e+01 6.15000e+02 1.65000e+03 1.40000e+03 2.39000e+03 6.00000e+03 8.00000e+02 4.85000e+02 1.99900e+03 9.80000e+03 1.94000e+02 9.90000e+02 8.50000e+02 1.80185e+05 3.32000e+02 3.99000e+02 1.32000e+03 2.38000e+02 2.76000e+02 3.41000e+02 3.08000e+02 5.70000e+02 2.40000e+03 2.09000e+02 8.75000e+02 5.90000e+03 2.90000e+01 3.81000e+02 2.52000e+02 3.50000e+03 2.02000e+02 3.91000e+02 4.65000e+03 7.48000e+02 5.50000e+03 4.90000e+02 5.55000e+02 3.26000e+02 8.90000e+03 1.99990e+04 7.85474e+05 5.75000e+02 4.75000e+02 4.04900e+03 1.16500e+03 4.67000e+02 3.86000e+02 4.65000e+02 9.60000e+02 2.07309e+05 4.40000e+02 1.04600e+03 2.88000e+02 7.25000e+02 7.95000e+02 2.36000e+02 4.71000e+02 3.20000e+03 1.12200e+03 5.98805e+05 4.00000e+00 1.60000e+03 3.03000e+02 7.80000e+02 5.80000e+02 3.38000e+02 6.94000e+02 2.71000e+02 2.11000e+02 3.64000e+02 2.39000e+02 2.78000e+02 1.06000e+03 3.42000e+02 9.80000e+02 5.60000e+03 3.07000e+02 5.10000e+02 3.88000e+02 7.79000e+02 2.06000e+02 3.76000e+02 3.63000e+02 3.59000e+02 3.14000e+02 2.90000e+04 1.04000e+03 2.70000e+01 2.53000e+02 1.37400e+03 3.17000e+02 9.00000e+00 2.46100e+03 2.13000e+02 3.89000e+02 2.44000e+02 3.85000e+02 1.69000e+03 4.45000e+02 7.20000e+02 1.25000e+05 2.93000e+02 5.84000e+03 5.30000e+02 2.33000e+02 3.22000e+02 4.88000e+02 9.15000e+02 6.23000e+02 4.55000e+02 2.60000e+01 3.57000e+02 2.83000e+02 3.09000e+02 1.14000e+03 3.12000e+02 2.96000e+02 4.03000e+02 3.55000e+02 4.97000e+02 5.34000e+02 3.01000e+02 1.86600e+03 5.45000e+02 4.95000e+02 6.80000e+02 1.01100e+03 3.43000e+02 1.62800e+03 5.01000e+02 8.40000e+02 2.70000e+03 2.87000e+02 7.30000e+02 1.70000e+03 2.42000e+02 1.11111e+05 8.50000e+03 1.41700e+03 3.08600e+03 2.25000e+03 8.65000e+02 2.31000e+02 2.56000e+02 2.94000e+02 3.67000e+02 6.33000e+02 1.30000e+03 4.08000e+02 6.70000e+02 2.92000e+02 1.38500e+03 2.97000e+02 1.70000e+01 4.37000e+02 1.86000e+03 4.01000e+02 5.44000e+02 7.10000e+02 5.58000e+02 6.75000e+02 1.15000e+03 8.10000e+02 3.04000e+02 5.95000e+02 6.54000e+02 5.99000e+02 2.66000e+02 4.34000e+02 5.13000e+02 9.30000e+02 3.10000e+01 3.44000e+02 3.02000e+02 2.20000e+03 4.19000e+02 3.58000e+02 2.82000e+02 3.78000e+02 3.24000e+02 6.39000e+02 4.29000e+02 5.85000e+02 3.54000e+02 9.95000e+02 7.39100e+03 3.38800e+03 4.18000e+02 5.24000e+02 4.28000e+02 5.22000e+02 7.57000e+02 5.56000e+02 4.94000e+02 3.27700e+03 4.47000e+02 9.09039e+05 6.87000e+02 3.36684e+05 4.30000e+03 7.70000e+02 1.33800e+03 2.84000e+02 2.41000e+02 3.98000e+02 7.50000e+03 1.68000e+03 2.54000e+02 5.64000e+02 3.72000e+03 2.08700e+03 9.19000e+02 5.09000e+02 1.18000e+03 4.03000e+03 3.00000e+04 1.38000e+03 1.95000e+03 2.68000e+02 1.76000e+03 1.49000e+03 5.18000e+02 6.10000e+02 3.45000e+03 1.10000e+01 1.40800e+03 3.33000e+02 1.04400e+03 2.95000e+03 2.20000e+05 4.38000e+02 9.36000e+02 3.29000e+02 9.48235e+05 8.20000e+02 1.10000e+05 5.87000e+02 5.90000e+02 3.27000e+02 1.75000e+03 5.04000e+02 4.62000e+02 3.72000e+02 3.18000e+02 1.23500e+03 5.76000e+02 3.21000e+02 6.11000e+02 7.90000e+02 6.27000e+02 1.25500e+03 6.48000e+02 1.02000e+03 5.92000e+02 2.77000e+02 2.79000e+02 7.28000e+02 6.09000e+03 4.86000e+02 1.01500e+03 4.22000e+02 3.11000e+02 6.45000e+02 9.75000e+02 2.73000e+02 3.46000e+02 7.16000e+02 5.65000e+02 4.13470e+04 1.00800e+03 5.14783e+05 1.20000e+05 3.82000e+02 8.27500e+03 3.37000e+02 5.68000e+02 2.62000e+02 1.13000e+03 4.24000e+02 3.68000e+02 4.53000e+02 2.69000e+02 1.11000e+03 2.43000e+02 1.40100e+03 2.61000e+02 3.31000e+02 3.60000e+03 1.65000e+05 5.52000e+02 2.58000e+02 4.81000e+02 1.74900e+03 6.05000e+02 3.00100e+03 6.00000e+00 3.79000e+02 1.83200e+03 1.90000e+01 1.44999e+05 5.47000e+02 4.41000e+02 8.60000e+03 6.91300e+03 3.17000e+03 2.51000e+02 1.20300e+03 2.91000e+02 1.17000e+03 1.15500e+03 7.38000e+02 3.16000e+02 4.42000e+02 4.30000e+05 3.52000e+02 1.75000e+05 1.40920e+04 2.60000e+05 1.48000e+03 2.60000e+03 4.56500e+03 2.35000e+03 1.00400e+03 5.69000e+02 6.95000e+02 1.45000e+03 4.50000e+03 2.83230e+04 6.88463e+05 1.73600e+03 1.29500e+03 5.95000e+03 9.85000e+03 3.10000e+03 5.05000e+02 1.37510e+04 4.63000e+02 1.40135e+05 1.02265e+05 1.31000e+03 1.10000e+04 7.12000e+02 1.51700e+03 5.73000e+02 9.14000e+02 1.37700e+03 1.25000e+03 1.79000e+03 4.89000e+02 5.03000e+02 3.30000e+03 7.75000e+02 1.80900e+03 6.07000e+02 4.70000e+03 6.59000e+02 1.30000e+04 4.77000e+02 1.19900e+03 4.66000e+04 5.96000e+02 1.42000e+03 6.47000e+02 1.80000e+04 1.51600e+03 4.33000e+02 5.28000e+02 2.97000e+03 7.58000e+02 2.75000e+03 1.82000e+03 1.51400e+03 4.59000e+02 1.51300e+03 8.80000e+03 6.68000e+02 4.07000e+02 7.59000e+02 1.22600e+03 1.54700e+03 4.56000e+02 1.61700e+03 1.51500e+03 3.97000e+02 6.08000e+02 1.88000e+03 6.84300e+03 7.77547e+05 1.97600e+03 6.25000e+02 2.81000e+02 3.15000e+03 2.31280e+05 5.23000e+02 8.27000e+02 1.76900e+03 5.41000e+02 8.89000e+02 6.56000e+02 1.46000e+03 1.40000e+05 1.44300e+03 6.96000e+02 2.92000e+03 2.45000e+03 1.71300e+03 1.52200e+03 4.83000e+02 7.37990e+05 1.68360e+05 1.69500e+03 1.07000e+03 4.79000e+02 4.69000e+02 4.14000e+02 1.01824e+05 3.96000e+02 3.51000e+02 8.58484e+05 5.27000e+02 4.27000e+02 1.14700e+03 4.80680e+04 4.90000e+03 9.50000e+03 9.94000e+02 1.08500e+03 6.36000e+02 5.86000e+02 4.04000e+02 1.99810e+04 5.01994e+05 5.00000e+05 2.00000e+05 6.26000e+02 9.41500e+03 8.32240e+04 1.05990e+04 7.01195e+05 1.51360e+04 9.98000e+02 4.61000e+02 4.97833e+05 2.11478e+05 7.51657e+05 5.13160e+04 9.02221e+05 1.97254e+05 4.14211e+05 1.81167e+05 1.87000e+03 6.76535e+05 8.60000e+02 4.17700e+03 9.40700e+03 8.87000e+02 9.30259e+05 7.19810e+04 8.06000e+02 2.99900e+03 4.99900e+03 1.36500e+03 6.15139e+05 1.70000e+04 3.00000e+05 5.93000e+02 4.46000e+02 2.24900e+03 6.49500e+03 1.85000e+04 2.99400e+03 6.73000e+02 1.36800e+03 4.58900e+03 7.84300e+03 1.69900e+03 1.61912e+05 1.42510e+04 6.48370e+04 2.60000e+04 1.15800e+04 8.90000e+02 8.49870e+04 1.61600e+03 4.21200e+03 2.56700e+03 1.64500e+03 1.99300e+05 1.55000e+03 1.30100e+03 1.11100e+03 4.76000e+02 6.02000e+02 9.00000e+04 7.26000e+02 5.88000e+02 3.53000e+02 1.53900e+03 8.48000e+02 7.61690e+04 3.80000e+03 1.60000e+01 1.44500e+03 3.55000e+03 6.99000e+02 8.78510e+04 3.30590e+04 6.84760e+04 6.34000e+02 1.40000e+01 8.69000e+02 1.74000e+03 2.96900e+03 1.94500e+03 6.25000e+03 4.13000e+02 1.46100e+03 3.01640e+04 8.70752e+05 6.15860e+04 3.31720e+04 2.00300e+03 7.95000e+03 9.95300e+03 5.53000e+02 8.59000e+03 3.93360e+05 8.45000e+03 6.79000e+02 7.41000e+02 2.34900e+03 5.88300e+03 8.91000e+02 3.59900e+03 8.08000e+02 6.53000e+02 5.49000e+02 7.99000e+02 9.47970e+04 6.95730e+04 1.40000e+04 6.82020e+04 2.71400e+04 5.29000e+02 8.23900e+03 1.00200e+03 3.94369e+05 4.78000e+02]Null: 146006------Column: heating_typeCounts: Kombi (Doğalgaz) 204150Klima 68197Merkezi Sistem (Isı Payı Ölçer) 30595Merkezi Sistem 22855Kalorifer (Doğalgaz) 10928Soba (Kömür) 8450Yerden Isıtma 6958Yok 6098Kat Kaloriferi 5655Kombi (Elektrikli) 3449Soba (Doğalgaz) 2602Güneş Enerjisi 1754Kalorifer (Kömür) 1503Jeotermal 1418Fancoil 495Kalorifer (Akaryakıt) 410Name: heating_type, dtype: int64Unique: ['Fancoil' 'Yok' 'Kalorifer (Doğalgaz)' 'Kalorifer (Kömür)' 'Kombi (Elektrikli)' 'Klima' 'Kombi (Doğalgaz)' 'Merkezi Sistem (Isı Payı Ölçer)' 'Merkezi Sistem' 'Soba (Kömür)' 'Yerden Isıtma' 'Soba (Doğalgaz)' 'Güneş Enerjisi' 'Kalorifer (Akaryakıt)' 'Jeotermal' 'Kat Kaloriferi' nan]Null: 27970------Column: priceCounts: 1000.0 9078 1500.0 5691 800.0 5425 1200.0 5415 750.0 5249 900.0 5083 1100.0 5064 350000.0 4631 250000.0 4605 850.0 4371 1300.0 4073 210000.0 3971 700.0 3926 185000.0 3775 1250.0 3775 175000.0 3632 220000.0 3625 300000.0 3605 165000.0 3588 260000.0 3545 225000.0 3528 450000.0 3465 230000.0 3359 275000.0 3354 235000.0 3297 200000.0 3251 1400.0 3188 265000.0 3088 650.0 3001 215000.0 2994 ... 382855.0 1 382800.0 1 93939.0 1 2990.0 1 187900.0 1 328500.0 1 1505000.0 1-250.0 1 753000.0 1 188280.0 1 1056000.0 1 188888.0 1 188900.0 1 378072.0 1 271500.0 1 189356.0 1 3030000.0 1 166250.0 1 69750.0 1 94800.0 1 189721.0 1 595.0 1 39000000.0 1 759999.0 1 380042.0 1 1521152.0 1 98400.0 1 95250.0 1 763000.0 1 189700.0 1Name: price, Length: 3703, dtype: int64Unique: [3.5000000e+03 4.9000000e+05 1.5500000e+05 ... 3.1234123e+07 1.1500610e+06 2.2230000e+05]Null: 715------Column: price_currencyCounts: TRY 400677EUR 922GBP 621USD 552Name: price_currency, dtype: int64Unique: ['TRY' 'GBP' 'EUR' 'USD' nan]Null: 715In [12]:
data.drop("type", axis = 1, inplace=True)In [13]:
data.shapeOut[13]:
(403487, 11)Обработка количества комнат
In [14]:
data['room_count_1']=data['room_count'].apply(lambda x: x.split('+')[0])data['room_count_1']=data['room_count_1'].replace(r'',0)data['room_count_2']=data['room_count'].apply(lambda x: x.split('+')[1])data['room_count'].apply(lambda x: x.split('+')[0])data['room_count_2']=data['room_count_2'].replace(r'',0)data['room_count_1']=data['room_count_1'].astype('int64')data['room_count_2']=data['room_count_2'].astype('int64')In [15]:
data.drop('room_count',axis=1,inplace=True)In [16]:
data.info()RangeIndex: 403487 entries, 0 to 403486Data columns (total 12 columns):sub_type 403487 non-null objectlisting_type 403487 non-null int64tom 403487 non-null int64building_age 376097 non-null objecttotal_floor_count 375466 non-null objectfloor_no 368191 non-null objectsize 257481 non-null float64heating_type 375517 non-null objectprice 402772 non-null float64price_currency 402772 non-null objectroom_count_1 403487 non-null int64room_count_2 403487 non-null int64dtypes: float64(2), int64(4), object(6)memory usage: 36.9+ MBIn [17]:
data.head()Out[17]:
sub_typelisting_typetombuilding_agetotal_floor_countfloor_nosizeheating_typepriceprice_currencyroom_count_1room_count_2
0Rezidans230020 ve üzeri290.0Fancoil3500.0TRY21
1Daire114020 ve üzeri20 ve üzeri43.0Fancoil490000.0TRY10
2Daire13001Yüksek GirişNaNFancoil155000.0TRY21
3Rezidans130320 ve üzeri20 ve üzeri450.0Fancoil32500000.0TRY61
4Rezidans130020 ve üzeri290.0Fancoil1450000.0TRY21
In [18]:
data["total_floor_count"] = data["total_floor_count"].replace(np.nan,0)In [19]:
data["total_floor_count"]=(data["total_floor_count"].astype('str').apply(lambda x:x.split(r" ")[0].split("-")[0])).astype('int64')In [20]:
data.head()Out[20]:
sub_typelisting_typetombuilding_agetotal_floor_countfloor_nosizeheating_typepriceprice_currencyroom_count_1room_count_2
0Rezidans230020290.0Fancoil3500.0TRY21
1Daire11402020 ve üzeri43.0Fancoil490000.0TRY10
2Daire13001Yüksek GirişNaNFancoil155000.0TRY21
3Rezidans13032020 ve üzeri450.0Fancoil32500000.0TRY61
4Rezidans130020290.0Fancoil1450000.0TRY21
In [21]:
data.info()RangeIndex: 403487 entries, 0 to 403486Data columns (total 12 columns):sub_type 403487 non-null objectlisting_type 403487 non-null int64tom 403487 non-null int64building_age 376097 non-null objecttotal_floor_count 403487 non-null int64floor_no 368191 non-null objectsize 257481 non-null float64heating_type 375517 non-null objectprice 402772 non-null float64price_currency 402772 non-null objectroom_count_1 403487 non-null int64room_count_2 403487 non-null int64dtypes: float64(2), int64(5), object(5)memory usage: 36.9+ MBIn [22]:
data["floor_no"].fillna("0", inplace=True)b = data["floor_no"].apply(lambda x: str(x).split(" ")[0])b = pd.to_numeric(b,errors="coerce")data["floor_no"]=b;In [23]:
data.head()Out[23]:
sub_typelisting_typetombuilding_agetotal_floor_countfloor_nosizeheating_typepriceprice_currencyroom_count_1room_count_2
0Rezidans2300202.090.0Fancoil3500.0TRY21
1Daire11402020.043.0Fancoil490000.0TRY10
2Daire13001NaNNaNFancoil155000.0TRY21
3Rezidans13032020.0450.0Fancoil32500000.0TRY61
4Rezidans1300202.090.0Fancoil1450000.0TRY21
In [24]:
data["floor_no"] = data["floor_no"].replace(np.nan,0)In [25]:
data["floor_no"]=data["floor_no"].astype("int64")In [26]:
data.info()RangeIndex: 403487 entries, 0 to 403486Data columns (total 12 columns):sub_type 403487 non-null objectlisting_type 403487 non-null int64tom 403487 non-null int64building_age 376097 non-null objecttotal_floor_count 403487 non-null int64floor_no 403487 non-null int64size 257481 non-null float64heating_type 375517 non-null objectprice 402772 non-null float64price_currency 402772 non-null objectroom_count_1 403487 non-null int64room_count_2 403487 non-null int64dtypes: float64(2), int64(6), object(4)memory usage: 36.9+ MBIn [27]:
data["building_age"].fillna("0", inplace=True)a = data["building_age"].apply(lambda x: x.split(" ")[0].split("-")[0])data["building_age"]=a.astype("int64")data["building_age"].value_counts()Out[27]:
0 1675646 5049511 3230916 313331 203554 1903221 184382 174663 156515 1358926 1058131 426836 134740 1059Name: building_age, dtype: int64In [28]:
data.info()RangeIndex: 403487 entries, 0 to 403486Data columns (total 12 columns):sub_type 403487 non-null objectlisting_type 403487 non-null int64tom 403487 non-null int64building_age 403487 non-null int64total_floor_count 403487 non-null int64floor_no 403487 non-null int64size 257481 non-null float64heating_type 375517 non-null objectprice 402772 non-null float64price_currency 402772 non-null objectroom_count_1 403487 non-null int64room_count_2 403487 non-null int64dtypes: float64(2), int64(7), object(3)memory usage: 36.9+ MBIn [29]:
data["price_currency"] = data["price_currency"].replace("TRY",0.11)data["price_currency"] = data["price_currency"].replace("GBP",1.1)data["price_currency"] = data["price_currency"].replace("USD",0.85)data["price_currency"] = data["price_currency"].replace("EUR",1)data["price"] = data["price"].replace(np.nan,0)data["price"]=data["price"].astype("int64")In [30]:
data["priceEUR"]=data["price"].replace(np.nan,0)data["priceEUR"]=data["price"] * data["price_currency"]data["priceEUR"]=data["price"].astype("int64")In [31]:
data.drop("price_currency",axis=1, inplace=True)In [32]:
data.head(100)Out[32]:
sub_typelisting_typetombuilding_agetotal_floor_countfloor_nosizeheating_typepriceroom_count_1room_count_2priceEUR
0Rezidans230020290.0Fancoil3500213500
1Daire1140202043.0Fancoil49000010490000
2Daire130010NaNFancoil15500021155000
3Rezidans13032020450.0Fancoil325000006132500000
4Rezidans130020290.0Fancoil1450000211450000
5Rezidans1302101045.0Fancoil78000011780000
6Daire25402014160.0Fancoil3750313750
7Villa192040NaNFancoil1500000411500000
8Daire111020140.0Fancoil1500000311500000
9Daire10110550.0Fancoil842562284256
10Müstakil Ev113010125.0Fancoil2450000312450000
11Rezidans21322020165.0Fancoil7500317500
12Daire130032NaNFancoil27000021270000
13Rezidans13001011NaNFancoil27000021270000
14Daire1158143NaNFancoil750003175000
15Villa294620200.0Fancoil3600413600
16Rezidans13008385.0Fancoil3800000113800000
17Daire2300202084.0Fancoil4200214200
18Rezidans2972101350.0Fancoil2000102000
19Rezidans115432020450.0Fancoil325000006132500000
20Daire104102NaNFancoil82934392829343
21Rezidans114122010NaNFancoil42000011420000
22Rezidans130210755.0Fancoil75000011750000
23Daire25211011100.0Fancoil1100211100
24Daire161032NaNFancoil27000021270000
25Daire13002020NaNFancoil1750000211750000
26Daire1118343100.0Fancoil28000021280000
27Rezidans1302101170.0Fancoil72000011720000
28Daire2300201630.0Fancoil1300101300
29Villa160040NaNFancoil1500000411500000
.......................................
70Villa130020180.0Fancoil1650000311650000
71Daire26012020120.0Fancoil3500213500
72Villa1138010NaNFancoil1100000311100000
73Villa1141030NaNFancoil125000003212500000
74Daire160033NaNFancoil16000011160000
75Rezidans130112020245.0Fancoil6500000316500000
76Rezidans21200101NaNFancoil2200112200
77Rezidans16402020NaNFancoil105000003110500000
78Daire1403140.0Fancoil42000011420000
79Rezidans19332020450.0Fancoil325000006132500000
80Daire191032NaNFancoil27000021270000
81Villa2124620200.0Fancoil3600413600
82Rezidans16332013173.0Fancoil4000000314000000
83Rezidans123112020320.0Fancoil270000003127000000
84Rezidans1302101365.0Fancoil98000021980000
85Daire212312020120.0Fancoil3500213500
86Daire16420120.0Fancoil42500031425000
87Villa134020230.0Fancoil1150000411150000
88Rezidans2170165NaNFancoil1200111200
89Daire10120922699.0Fancoil16222263162222
90Daire130320NaNFancoil56000041560000
91Rezidans13032020400.0Fancoil325000006132500000
92Daire230020250.0Fancoil2500112500
93Daire16110060.0Fancoil50000095500000
94Rezidans10280222.0Fancoil3433501223433501
95Müstakil Ev11301130NaNFancoil31000031310000
96Daire125051130.0Fancoil44900031449000
97Rezidans13102020NaNFancoil105000003110500000
98Müstakil Ev130230NaNFancoil62500051625000
99Yazlık1461109150.0Fancoil50000031500000
100 rows × 12 columns
In [33]:
data.info()RangeIndex: 403487 entries, 0 to 403486Data columns (total 12 columns):sub_type 403487 non-null objectlisting_type 403487 non-null int64tom 403487 non-null int64building_age 403487 non-null int64total_floor_count 403487 non-null int64floor_no 403487 non-null int64size 257481 non-null float64heating_type 375517 non-null objectprice 403487 non-null int64room_count_1 403487 non-null int64room_count_2 403487 non-null int64priceEUR 403487 non-null int64dtypes: float64(1), int64(9), object(2)memory usage: 36.9+ MBIn [34]:
data_dummies=pd.get_dummies(data.select_dtypes("object"))In [35]:
data_all=pd.concat([data_dummies,data.select_dtypes("int64"), data.select_dtypes("float64")],axis=1)In [36]:
data_all.shapeOut[36]:
(403487, 38)In [37]:
data_all['size'].fillna('0', inplace=True)In [38]:
data_all.head()Out[38]:
sub_type_Dairesub_type_Komple Binasub_type_Kooperatifsub_type_Köşk / Konak / Yalısub_type_Loftsub_type_Müstakil Evsub_type_Prefabrik Evsub_type_Rezidanssub_type_Villasub_type_Yalı Dairesi...listing_typetombuilding_agetotal_floor_countfloor_nopriceroom_count_1room_count_2priceEURsize
00000000100...2300202350021350090
11000000000...114020204900001049000043
21000000000...130010155000211550000
30000000100...13032020325000006132500000450
40000000100...1300202145000021145000090
5 rows × 38 columns
Модуль B: Классификация
Отбор признаков
In [39]:
plt.style.use("fivethirtyeight")plt.figure(figsize=(10,8))sns.heatmap(data.corr(), annot = True, fmt = ".2f");
In [40]:
corr_matrix = data.corr().abs()In [41]:
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k = 1).astype(np.bool))In [42]:
to_drop = [column for column in upper.columns if any (upper[column]>0.6)]In [43]:
to_dropOut[43]:
['floor_no', 'priceEUR']In [44]:
data.drop(to_drop, axis = 1, inplace = True)In [47]:
plt.style.use("fivethirtyeight")plt.figure(figsize=(10,8))sns.heatmap(data.corr(), annot = True, fmt = ".2f", linecolor='white');
In [48]:
from sklearn.ensemble import RandomForestClassifierfrom sklearn.model_selection import train_test_splitIn [49]:
data.info()RangeIndex: 403487 entries, 0 to 403486Data columns (total 10 columns):sub_type 403487 non-null objectlisting_type 403487 non-null int64tom 403487 non-null int64building_age 403487 non-null int64total_floor_count 403487 non-null int64size 257481 non-null float64heating_type 375517 non-null objectprice 403487 non-null int64room_count_1 403487 non-null int64room_count_2 403487 non-null int64dtypes: float64(1), int64(7), object(2)memory usage: 30.8+ MBIn [50]:
feature = data[['listing_type', 'tom', 'total_floor_count', 'size', 'price', 'room_count_1', 'room_count_2']]target = data['building_age']In [51]:
feature_train, feature_test, target_train, target_test = train_test_split(feature, target, test_size = 0.3, random_state = 0)In [52]:
plt.style.use("fivethirtyeight")plt.figure(figsize=(10,8))sns.heatmap(data_all[['listing_type', 'tom', 'building_age', 'total_floor_count',\ 'room_count_1', 'room_count_2', 'size', 'price']].corr(), annot = True, fmt = ".2f");
In [53]:
data_all['listing_type'].unique()Out[53]:
array([2, 1, 3])In [54]:
data_all.drop('price', axis=1, inplace=True)In [55]:
plt.scatter(data_all['building_age'],data_all['listing_type'])Out[55]:
In [56]:
plt.scatter(data_all['building_age'],data_all['sub_type_Komple Bina'])Out[56]:
In [57]:
plt.scatter(data_all['building_age'],data_all['priceEUR'])Out[57]:
In [58]:
plt.hist(data_all['building_age']);
In [ ]:
In [59]:
# Create a list of buildings with more than 100 measurementstypes = data.dropna(subset=['building_age'])types = types['listing_type'].value_counts()types = list(types[types.values > 100].index)# Plot of distribution of scores for building categoriesplt.figure(figsize=(12,10))# Plot each buildingfor b_type in types: # Select the building type subset = data[data['listing_type'] == b_type] # Density plot of Energy Star scores sns.kdeplot(subset['building_age'].dropna(), label = b_type, shade = False, alpha = 0.8); # label the plotplt.xlabel("Energy Star Score", size = 20)plt.ylabel("Density", size = 20); plt.title('Density Plot of Energy Star Scores by Building Type', size = 28);
In [60]:
data_all.columnsOut[60]:
Index(['sub_type_Daire', 'sub_type_Komple Bina', 'sub_type_Kooperatif', 'sub_type_Köşk / Konak / Yalı', 'sub_type_Loft', 'sub_type_Müstakil Ev', 'sub_type_Prefabrik Ev', 'sub_type_Rezidans', 'sub_type_Villa', 'sub_type_Yalı Dairesi', 'sub_type_Yazlık', 'sub_type_Çiftlik Evi', 'heating_type_Fancoil', 'heating_type_Güneş Enerjisi', 'heating_type_Jeotermal', 'heating_type_Kalorifer (Akaryakıt)', 'heating_type_Kalorifer (Doğalgaz)', 'heating_type_Kalorifer (Kömür)', 'heating_type_Kat Kaloriferi', 'heating_type_Klima', 'heating_type_Kombi (Doğalgaz)', 'heating_type_Kombi (Elektrikli)', 'heating_type_Merkezi Sistem', 'heating_type_Merkezi Sistem (Isı Payı Ölçer)', 'heating_type_Soba (Doğalgaz)', 'heating_type_Soba (Kömür)', 'heating_type_Yerden Isıtma', 'heating_type_Yok', 'listing_type', 'tom', 'building_age', 'total_floor_count', 'floor_no', 'room_count_1', 'room_count_2', 'priceEUR', 'size'], dtype='object')In [61]:
# Create a list of buildings with more than 100 measurementstypes = data.dropna(subset=['building_age'])types = types['room_count_2'].value_counts()types = list(types[types.values > 100].index)# Plot of distribution of scores for building categoriesplt.figure(figsize=(12,10))# Plot each buildingfor b_type in types: # Select the building type subset = data[data['room_count_2'] == b_type] # Density plot of Energy Star scores sns.kdeplot(subset['building_age'].dropna(), label = b_type, shade = False, alpha = 0.8); # label the plotplt.xlabel("Energy Star Score", size = 20)plt.ylabel("Density", size = 20); plt.title('Density Plot of Energy Star Scores by Building Type', size = 28);
115
0
