0000_PREPROCESSING_DATA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>publication</th>\n",
       "      <th>author</th>\n",
       "      <th>date</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>url</th>\n",
       "      <th>content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>17283</td>\n",
       "      <td>House Republicans Fret About Winning Their Hea...</td>\n",
       "      <td>New York Times</td>\n",
       "      <td>Carl Hulse</td>\n",
       "      <td>2016-12-31</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>WASHINGTON  —   Congressional Republicans have...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>17284</td>\n",
       "      <td>Rift Between Officers and Residents as Killing...</td>\n",
       "      <td>New York Times</td>\n",
       "      <td>Benjamin Mueller and Al Baker</td>\n",
       "      <td>2017-06-19</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>After the bullet shells get counted, the blood...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>17285</td>\n",
       "      <td>Tyrus Wong, ‘Bambi’ Artist Thwarted by Racial ...</td>\n",
       "      <td>New York Times</td>\n",
       "      <td>Margalit Fox</td>\n",
       "      <td>2017-01-06</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>When Walt Disney’s “Bambi” opened in 1942, cri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>17286</td>\n",
       "      <td>Among Deaths in 2016, a Heavy Toll in Pop Musi...</td>\n",
       "      <td>New York Times</td>\n",
       "      <td>William McDonald</td>\n",
       "      <td>2017-04-10</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Death may be the great equalizer, but it isn’t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>17287</td>\n",
       "      <td>Kim Jong-un Says North Korea Is Preparing to T...</td>\n",
       "      <td>New York Times</td>\n",
       "      <td>Choe Sang-Hun</td>\n",
       "      <td>2017-01-02</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>SEOUL, South Korea  —   North Korea’s leader, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49995</th>\n",
       "      <td>53287</td>\n",
       "      <td>73465</td>\n",
       "      <td>Rex Tillerson Says Climate Change Is Real, but …</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>Robinson Meyer</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>As chairman and CEO of ExxonMobil, Rex Tillers...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49996</th>\n",
       "      <td>53288</td>\n",
       "      <td>73466</td>\n",
       "      <td>The Biggest Intelligence Questions Raised by t...</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>Amy Zegart</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>I’ve spent nearly 20 years looking at intellig...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49997</th>\n",
       "      <td>53289</td>\n",
       "      <td>73467</td>\n",
       "      <td>Trump Announces Plan That Does Little to Resol...</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>Jeremy Venook</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Donald Trump will not be taking necessary st...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>53290</td>\n",
       "      <td>73468</td>\n",
       "      <td>Dozens of For-Profit Colleges Could Soon Close</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>Emily DeRuy</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Dozens of   colleges could be forced to close ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>53291</td>\n",
       "      <td>73469</td>\n",
       "      <td>The Milky Way’s Stolen Stars</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>Marina Koren</td>\n",
       "      <td>2017-01-11</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>The force of gravity can be described using a ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50000 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0     id                                              title  \\\n",
       "0               0  17283  House Republicans Fret About Winning Their Hea...   \n",
       "1               1  17284  Rift Between Officers and Residents as Killing...   \n",
       "2               2  17285  Tyrus Wong, ‘Bambi’ Artist Thwarted by Racial ...   \n",
       "3               3  17286  Among Deaths in 2016, a Heavy Toll in Pop Musi...   \n",
       "4               4  17287  Kim Jong-un Says North Korea Is Preparing to T...   \n",
       "...           ...    ...                                                ...   \n",
       "49995       53287  73465   Rex Tillerson Says Climate Change Is Real, but …   \n",
       "49996       53288  73466  The Biggest Intelligence Questions Raised by t...   \n",
       "49997       53289  73467  Trump Announces Plan That Does Little to Resol...   \n",
       "49998       53290  73468    Dozens of For-Profit Colleges Could Soon Close    \n",
       "49999       53291  73469                       The Milky Way’s Stolen Stars   \n",
       "\n",
       "          publication                         author        date    year  \\\n",
       "0      New York Times                     Carl Hulse  2016-12-31  2016.0   \n",
       "1      New York Times  Benjamin Mueller and Al Baker  2017-06-19  2017.0   \n",
       "2      New York Times                   Margalit Fox  2017-01-06  2017.0   \n",
       "3      New York Times               William McDonald  2017-04-10  2017.0   \n",
       "4      New York Times                  Choe Sang-Hun  2017-01-02  2017.0   \n",
       "...               ...                            ...         ...     ...   \n",
       "49995        Atlantic                 Robinson Meyer  2017-01-11  2017.0   \n",
       "49996        Atlantic                     Amy Zegart  2017-01-11  2017.0   \n",
       "49997        Atlantic                  Jeremy Venook  2017-01-11  2017.0   \n",
       "49998        Atlantic                    Emily DeRuy  2017-01-11  2017.0   \n",
       "49999        Atlantic                   Marina Koren  2017-01-11  2017.0   \n",
       "\n",
       "       month  url                                            content  \n",
       "0       12.0  NaN  WASHINGTON  —   Congressional Republicans have...  \n",
       "1        6.0  NaN  After the bullet shells get counted, the blood...  \n",
       "2        1.0  NaN  When Walt Disney’s “Bambi” opened in 1942, cri...  \n",
       "3        4.0  NaN  Death may be the great equalizer, but it isn’t...  \n",
       "4        1.0  NaN  SEOUL, South Korea  —   North Korea’s leader, ...  \n",
       "...      ...  ...                                                ...  \n",
       "49995    1.0  NaN  As chairman and CEO of ExxonMobil, Rex Tillers...  \n",
       "49996    1.0  NaN  I’ve spent nearly 20 years looking at intellig...  \n",
       "49997    1.0  NaN    Donald Trump will not be taking necessary st...  \n",
       "49998    1.0  NaN  Dozens of   colleges could be forced to close ...  \n",
       "49999    1.0  NaN  The force of gravity can be described using a ...  \n",
       "\n",
       "[50000 rows x 10 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_1 = pd.read_csv('data/articles1.csv')\n",
    "df_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1 = df_1[df_1['content'].notna()]\n",
    "df_1 = df_1.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles = df_1['title'].array\n",
    "papers = df_1['content'].array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Rift Between Officers and Residents as Killings Persist in South Bronx - The New York Times'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titles[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'After the bullet shells get counted, the blood dries and the votive candles burn out, people peer down from   windows and see crime scenes gone cold: a band of yellow police tape blowing in the breeze. The South Bronx, just across the Harlem River from Manhattan and once shorthand for urban dysfunction, still suffers violence at levels long ago slashed in many other parts of New York City. And yet the city’s efforts to fight it remain splintered, underfunded and burdened by scandal. In the 40th Precinct, at the southern tip of the Bronx, as in other poor, minority neighborhoods across the country, people long hounded for   infractions are crying out for more protection against grievous injury or death. By September, four of every five shootings in the precinct this year were unsolved. Out of the city’s 77 precincts, the 40th has the highest murder rate but the fewest detectives per violent crime, reflecting disparities in staffing that hit hardest in some neighborhoods outside Manhattan, according to a New York Times analysis of Police Department data. Investigators in the precinct are saddled with twice the number of cases the department recommends, even as their bosses are called to Police Headquarters to answer for the sharpest crime rise in the city this year. And across the Bronx, investigative resources are squeezed. It has the highest   rate of the city’s five boroughs but the thinnest detective staffing. Nine of the 14   precinct detective squads for violent crime in the city are there. The borough’s robbery squad is smaller than Manhattan’s, even though the Bronx has had 1, 300 more cases this year. And its homicide squad has one detective for every four murders, compared with one detective for roughly every two murders in Upper Manhattan and more than one detective per murder in Lower Manhattan. In   lobbies and   family apartments, outside methadone clinics and art studios, people take note of the inequity. They hear police commanders explain that they lack the resources to place a floodlight on a dangerous block or to post officers at a   corner. They watch witnesses cower behind   doors, more fearful of a gunman’s crew than confident in the Police Department’s ability to protect them. So though people see a lot, they rarely testify. And in the South Bronx, as in so many predominantly black and Hispanic neighborhoods like it in the United States, the contract between the police and the community is in tatters. Some people have stories of crime reports that were ignored, or 911 calls that went unanswered for hours. Others tell of a 911 call for help ending in the caller’s arrest, or of a minor charge leading to 12 hours in a fetid holding cell. This is the paradox of policing in the 40th Precinct. Its neighborhoods have historically been prime targets for aggressive tactics, like    that are designed to ward off disorder. But precinct detectives there have less time than anywhere else in the city to answer for the blood spilled in violent crimes. Gola White, who was beside her daughter when she was shot and killed in a playground this summer, four years after her son was gunned down in the same housing project, ticked off the public safety resources that she said were scant in Bronx neighborhoods like hers: security cameras, lights, locks, investigating police officers. “Here, we have nothing,” she said. When it comes to “  families,” she said, the authorities “don’t really care as much. That’s how I feel. ” The Times has been documenting the murders logged this year in the 40th Precinct, one of a handful of neighborhoods where deadly violence remains a problem in an era of   crime in New York City. The homicides  —   14 in the precinct this year, up from nine in 2015  —   strain detectives, and when they go unsolved, as half of them have this year, some look to take the law into their own hands. From hundreds of conversations with grieving relatives and friends, witnesses and police officers, the social forces that flare into murder in a place like the 40th Precinct become clearer: merciless gang codes, mental illness, drugs and long memories of feuds that simmered out of officers’ view. The reasons some murders will never be solved also emerge: paralyzing fear of retribution, victims carrying secrets to their graves and relentless casework that forces detectives to move on in hopes that a break will come later. Frustrations build on all sides. Detectives’ phones rarely ring with tips, and officers grow embittered with witnesses who will not cooperate. In the meantime, a victim’s friends conduct their own investigations, and talk of grabbing a stash gun from a wheel well or a mother’s apartment when they find their suspect. In the chasm between the police and the community, gangs and gun violence flourish. Parents try to protect their families from drug crews’ threats, and officers work to overcome the residue of years of mistrust and understaffing in communities where they still go racing from one 911 call to the next. The streets around St. Mary’s Park were the scene of two fatal shootings logged in the 40th Precinct this year. Both are unsolved. James Fernandez heard talk of the murders through the door of his   apartment on East 146th Street in a     the Betances Houses. He lived at the end of a long hallway strewn with hypodermic needles, empty dope bags and discarded Hennessy bottles. A   young men who spoke of being in a subset of the Bloods gang had made it their drug market, slinging marijuana and cocaine to regulars, flashing firearms and blowing smoke into the Fernandez apartment. When Mr. Fernandez, 40, asked the young men to move, they answered by busting up his car. This kind of crime, an anachronism in much of New York, still rattles the 40th Precinct, even though murders there have fallen to 14 this year from 83 in 1991. It has more major felony crimes per resident than any other residential district in the city. It is also one of the poorest communities in the country, and many young men find their way into underground markets. Mr. Fernandez was not one to shrink from the threats. When he was growing up on the Lower East Side, he rode his bicycle around to the customers of the drug dealers he worked for and collected payments in a backpack. After leaving that life, he got a tech maintenance job and, three years ago, moved into the Betances Houses with his wife and daughter, now 11. He had two choices to get help with the drug crew: call the police for help and risk being labeled a snitch, or call his old Lower East Side bosses for muscle and risk violence. He chose the police. Again and again, he walked into a local substation, Police Service Area 7, and asked for protection. His daughter was using an inhaler to relieve coughs from the marijuana smoke. Mr. Fernandez and his wife got terrible headaches. “There’s a lot of killers here, and we are going to kill you,” a sergeant’s police report quoted a    telling Mr. Fernandez in August 2015. A second report filed the same day said a    warned him, “I’m going to shoot through your window. ” Mr. Fernandez told the police both the teenagers’ names, which appear in the reports, and then went home. He said one of their friends had seen him walk into the substation, and they tried to intimidate him out of filing another report. Three days later, the same    propped his bike on their door, “then said if I was to open the door and say something, they would body slam me,” Mr. Fernandez’s wife, Maria Fernandez, wrote on slips of paper she used to document the hallway ruckus and the inadequate police response. The boys made comments about how easy a target she was and about how they would have to “slap” her if she opened the door while they made a drug sale, and they threatened to beat the Fernandez family because “they are the ones snitching,” her notes say. But another   complaint at the substation, 10 days after the first, brought no relief. A week later, feeling desperate, Ms. Fernandez tried calling: first to the substation, at 8:50 p. m. when one of the boys blew weed smoke at her door and made a   threat to attack her, and then to 911 at 10:36 p. m. The police never came, she wrote in her notes. She tried the 40th Precinct station house next, but officers at the desk left her standing in the public waiting area for a   she said, making her fear being seen again. Officers put her in worse danger some months later, she said, when they came to her door and announced in front of the teenagers that they were there on a complaint about drug activity. Mr. Fernandez started doing the work that he said the police had failed to do. He wired a camera into his peephole to record the drugs and guns. The footage hark back to the New York of the 1980s, still very much present to some of the precinct’s residents. Around 6:30 each morning, Sgt. Michael J. LoPuzzo walks through the tall wooden doors of the 40th Precinct station house. The cases that land on his metal desk  —   dead bodies with no known cause, strip club brawls, shooting victims hobbling into the hospital themselves  —   bring resistance at every turn, reminding him of an earlier era in the city’s   campaign. “I haven’t got one single phone call that’s putting me in the right direction here,” said Sergeant LoPuzzo, the head of the precinct’s detective squad, one day this summer as he worked on an answer to an email inquiry from a murder victim’s aunt about why the killer had not been caught. “And people just don’t understand that. ” Often it is detectives who most feel the effects of people turning on the police. Witnesses shout them away from their doors just so neighbors know they refuse to talk. Of the 184 people who were shot and wounded in the Bronx through early September, more than a third  —   66 victims  —   refused to cooperate. Over the same period in the 40th Precinct, squad detectives closed three of 17 nonfatal shootings, and 72 of 343 robbery cases. Part of the resistance stems from   preventive policing tactics, like    that were a hallmark of the     style under former Mayor Michael R. Bloomberg and his police commissioner, Raymond W. Kelly. Near the height of the    strategy, in 2012, the 40th Precinct had the   stops in the city, the   stops in which officers used force and the most frisks. Of 18, 276 stops that year, 15, 521 were of people who had done nothing criminal. The precinct was also one of the   areas that the department flooded with its newest officers. At roll calls, they were pressured to generate numbers: write tickets and make arrests. They had no choice but to give a summons to a young man playing in a park after dark, even if the officers had done the same growing up in the same neighborhood. “I need to bring something in today to justify my existence,” Officer Argenis Rosado, who joined the precinct in 2010, said in an interview at the station house. “So now you’re in a small area, and day after day you’re hammering the same community. Of course that community’s eventually going to turn on you. ” The pressure warped the way officers and residents saw each other. Rookies had to ignore why someone might be drinking outside or sitting on a stoop. “Some of the cops that came out at that time probably viewed the community differently, too,” said Hector Espada, a   veteran of the precinct. “Not because they wanted to, but because they had to. Because some way or somehow, you can’t give someone a $115 summons and feel like you guys could still have a civil conversation after that. ” Morale wilted in the aged station house on Alexander Avenue, in Mott Haven. Officers felt pressure to downgrade crime complaints to make them appear less serious. Several said in interviews that they had overlooked crime reports from immigrants because they were seen as unlikely to complain, and watched supervisors badger victims into repeating their stories in hopes that they would drop their complaints. The practice of downgrading complaints resulted in the disciplining of 19 officers in the precinct last year, one in a string of scandals that has left officers there feeling overscrutinized for problems that also existed elsewhere. Four commanders in the precinct were sent packing in five years, one of them after officers were found to be “ticket fixing,” or forgiving parking tickets for friends, and another after he was recorded giving guidance on whom to stop and frisk: black boys and men, ages 14 to 21. Some officers fled to other commands. Others became reluctant to take assignments in proactive policing units, like   that put them in   situations on the street. “Whenever I walked through the doors of the precinct, to me, it seemed like a black cloud,” said Russell Lewis, a    of the 40th. “It was like a heaviness. When you walked in, all you wanted to do was do your 8 hours 35 minutes and go home, because you didn’t want to get caught up in anything. ” The precinct covers only about two square miles, but the more than a dozen housing projects there mean that it overflows with people. Methadone clinics draw addicts from around the city.   lofts on the southern edge of the precinct presage a wave of gentrification. Even as the Police Department has hired 1, 300 more officers for neighborhood policing and counterterrorism, officers in the 40th Precinct said they could still rush to 25 911 calls during a shift  —   a number unchanged from what the new police commissioner, James P. O’Neill, said he was handling in a similar South Bronx precinct 15 years ago. Several dozen calls at a time can be waiting for a response. Residents know that if you want the police for a domestic problem, it helps to hint that there is a weapon. Last year, the precinct drew the   number of civilian complaints for officer misconduct in the city, and the most lawsuits stemming from police actions. The precinct is trying to improve morale under a new commanding officer, Deputy Inspector Brian Hennessy. A cadre of what the department calls neighborhood coordination officers has been on patrol since last January, part of a citywide effort under Mr. O’Neill and Mayor Bill de Blasio to bring back the beat cop, unencumbered by chasing every last 911 call, who can listen to people’s concerns and help with investigations. The precinct has made among the most gun arrests in the city, and officers said they now had more discretion to resolve encounters without a summons or an arrest. At one corner near a school, on Courtlandt Avenue and East 151st Street, that has long spawned complaints about gunfire and fights, Inspector Hennessy and some of his officers painted over graffiti and swept up drug paraphernalia this summer. People said it was the first answer to their complaints in years. But the inspector acknowledged that the residue of   policing lingers. “That perception really sticks,” he said. The workload in the 40th Precinct is startling and reveals a gap in how detective squads are equipped to answer violent crime in Manhattan compared with the Bronx, Brooklyn and Queens. Three of the precinct’s 16 detectives are carrying more than 400 cases each this year, and many others have loads in the high 300s, even though the department advises 150 in violent precincts. When they are assigned a homicide, they typically have four days to investigate before dealing with other cases. Quieter precincts can give detectives a month with little distraction to investigate a murder. Detectives in the 40th Precinct have each handled an average of 79 violent felonies this year through    —   murders, rapes, felony assaults and robberies. By contrast, a detective in the precinct on the southern end of Staten Island carries nine such cases a detective in the precinct patrolling Union Square and Gramercy Park handles 16 and a detective in the precinct for most of Washington Heights handles 32, the citywide median. Last year, the 40th was the    for violent crime, with 65 cases per detective. In the Bronx as a whole, a precinct detective has carried an average of 58 violent felonies this year, compared with 27 in Manhattan, 37 in Brooklyn, 38 in Queens and 25 on Staten Island. Rape cases and robbery patterns are later sent to more specialized units, but precinct detectives do extensive initial work to interview victims, write reports and process evidence. Precincts in much of Manhattan, which are whiter and wealthier than the South Bronx, often have more property felonies, like stolen laptops or credit cards, and the police say those can be complex. But even accounting for those crimes, the 40th Precinct has some of the heaviest caseloads of overall crime per detective in the city. Michael Palladino, the head of the Detectives’ Endowment Association and a former Bronx officer, said staffing disparities affected the department’s efforts to build trust in communities like the South Bronx. Witnesses make a calculation, he said: “If I cooperate with the detectives, there’s so much work, there’s so few of them there, they won’t even get the chance to protect me, or they’ll be there too late when the retaliation comes. ” Sergeant LoPuzzo, who turned down a more prestigious post to stay in the 40th Precinct, said that his squad worked tirelessly to handle cases with the people he had, and that while every squad wanted more detectives, staffing needs for counterterrorism units and task forces had created new deployment challenges across the department. “We fight with the army we have, not the army we wish we had,” he said. Details of how the Police Department assigns its 36, 000 officers are closely held and constantly in flux, and the public has minimal information on how personnel are allocated. Presented with The Times’s analysis of confidential staffing data, the department’s chief of detectives, Robert K. Boyce, vowed to send more detectives to the 40th Precinct and said the department would reassess its deployment more broadly in troubled precincts. He said a recent decision to bring gang, narcotics and vice detectives under his command made it easier to shift personnel. Chief Boyce said the burdens on detectives went beyond felony crimes to include   and   cases. And he noted the support that precinct squads got from centralized units focusing on robberies, gangs or grand larcenies, for example. Major crime keeps pounding the 40th Precinct, at rates that in 2015 were only a tenth of a percent lower than in 2001, even as citywide crime dropped by more than a third over the same period. But the precinct’s detective squad shrank by about eight investigators during those years, according to staffing data obtained from the City Council through a Freedom of Information Law request. The squad covering Union Square and Gramercy Park, where crime dropped by a third over that period, grew by about 11 investigators. (The 40th Precinct was given an additional detective and four   investigators this summer, when it was already missing three detectives for illness or other reasons.) Retired detectives are skeptical that community relations alone can drive down crime in the city’s last “” the busiest precincts. Rather, they say, the Police Department should be dedicating more resources to providing the same sort of robust investigative response that seems standard in Manhattan. “Any crime in Manhattan has to be solved,” said Howard Landesberg, who was a 40th Precinct detective in the late 1980s. “The outer boroughs are, like, forgotten. ” Retired detectives said that understaffing made it harder to solve crimes in the Bronx, Brooklyn and Queens, where the higher prevalence of gang and drug killings already saddled investigators with cases in which people were not inclined to cooperate. Through   detectives had closed 67 percent of homicides in Manhattan and 76 percent of those in Staten Island this year, compared with 54 percent of those in the Bronx, 42 percent of those in Queens and 31 percent of those in Brooklyn. Of last year’s homicides, detectives cleared 71 percent in Manhattan, 63 percent in the Bronx, 62 percent in Queens, 57 percent in Staten Island and 31 percent in Brooklyn. “It’s the culture of the Police Department that they worry about Manhattan,” said Joseph L. Giacalone, a former sergeant in the Bronx Cold Case Squad, in part “because that’s where the money is. ” He added: “When de Blasio came in, he talked about the tale of two cities. And then he’s done the complete opposite of what he said. It’s just business as usual. ” The Bronx’s struggles extend into prosecutions. In each of the last five years, prosecutors in the Bronx have declined to prosecute violent felony cases more than anywhere else in the city. And the rate of conviction in the Bronx is routinely the lowest in the city as well, but has ticked up this year to surpass Brooklyn’s rate through November as Bronx prosecutors work to streamline cases. Some cases have become even more difficult to win because of the   problem in the 40th Precinct, which has allowed defense lawyers to attack the credibility of officers who were implicated, said Patrice O’Shaughnessy, a spokeswoman for the Bronx District Attorney’s office. The district attorney, Darcel D. Clark, elected in 2015, said in a statement, “I was a judge here in the Bronx, and I heard from jurors that they can’t be impartial because they don’t trust the police. ” Against that tide of mistrust, Sergeant LoPuzzo’s detectives work 36 hours straight on some fresh cases. They buy Chinese takeout with their own money for a murder suspect. They carry surveillance videos home in hopes that their personal computers may enhance them better than a squad computer. They buy an urn for a homeless mother who has her murdered son’s ashes in a box. In the months after a killing, they can seem like the only people in this glittering city who are paying attention to the 40th Precinct’s homicide victims. Newly fatherless children go back to school without a therapist’s help. Victims’ families wander confused through a courthouse and nearly miss an appearance. Newspapers largely ignore killings of people with criminal pasts, pushing them down the priority lists of the   chiefs at Police Headquarters. In a stuffy   squad room, the detectives of the 40th Precinct grapple with an inheritance of government neglect. They meet mothers who believe their sons might never have been murdered had a city guidance counselor listened to pleas to help them stay enrolled, or had a city housing worker fixed the locks or lights on a building. And the detectives work alongside a vicious system on the streets for punishing police cooperators. Young men scan court paperwork in prison, looking for the names of people who turned on them. One murder victim in the precinct this year was cast out of his crew after he avoided being arrested with them in a gang takedown some believed he was cooperating. A longtime 40th Precinct detective, Jeff Meenagh, said a witness in a homicide case was going to testify until he went back to his neighborhood and was told that anyone who testified would “get what you deserve. ” The allies Sergeant LoPuzzo makes are friendly only for so long. He helped clear a woman’s son of a robbery charge by locating surveillance video that proved he was not the robber. The mother started calling with tips under a code name  —   about a gun under a car, for example. But she always refused to testify. And she cut ties this year after Sergeant LoPuzzo arrested her son in the stabbing of two people and her   in a shooting. New York City owns East 146th Street and the   buildings on each side. But James Fernandez, in the Betances Houses, said the reality on the ground was different: The drug boss ran the block. By October, Mr. Fernandez was increasingly afraid  —   and fed up. Mr. Fernandez and his wife went so far as to give officers keys to the building door, so they could get in whenever they wanted, showed them the videos and offered them   access to his camera so they could see what was happening in the hallway. A couple of officers said they needed a supervisor’s permission to do more. Others answered that the young men were only making threats. Officers occasionally stopped outside their building, causing the young men to scatter, but did not come inside, Mr. Fernandez said. The menacing worsened. Mr. Fernandez’s daughter was harassed as she arrived home from school. She grew more and more distressed, and her parents had her start seeing a therapist. Mr. Fernandez made several complaints at the office of the borough president, Ruben Diaz Jr. and visited a victim’s advocate in the district attorney’s office. On Oct. 20, 2015, he sent an online note to the police commissioner’s office. “We went to all proper channels for help,” the note said. “Both precincts failed us, except 2 officers who helped us, but their hands are tied. No one else to turn to. I have months of video of multiple crimes taking place and we are in extreme danger. ” “40th and PSA 7 won’t do anything,” he wrote, referring to the local substation. “Please we need to speak to some one with authority. ” The local substation commander, Deputy Inspector Jerry O’Sullivan, and the Bronx narcotics unit were alerted to the complaints. But Mr. Fernandez said he never heard from them. So he relied on his own street instincts to protect his family. He made pleas to a man he thought was employing the dealers in the hallway. The activity quieted briefly, but it returned after the young men rented a room in a woman’s apartment upstairs. Mr. Fernandez approached a different man who he learned was the boss of the operation. The man agreed to ask the dealers to calm down. He even hired a drug customer to sweep the hallway, Mr. Fernandez said. But two weeks later, the dealing and the harassment resumed. So he went to his old Lower East Side bosses, who hired men to trail his wife and daughter on their way out of the building and make sure they made it safely to school. At other times they sat outside the Betances Houses. He also bought two bulletproof vests, for about $700 each. He could not find one small enough for his daughter. “I have no faith in the City of New York, I have no faith in the police, I have no faith in the politicians,” Mr. Fernandez said. “The only thing I know for sure: God, if we’re in a situation again, I will be left to defend my family. ” Paying such close attention to what was happening in the hallway, Mr. Fernandez said he learned some details about two recent homicides that the 40th Precinct was investigating. But because his calls for help were going nowhere, he said he decided not to put himself in greater risk by talking: He would not tell the police what he had learned. “I’m bending over backward, and nobody’s not even doing anything,” he said. “Why am I going to help you, if you ain’t going to help me?” By last January, a new neighborhood coordination officer was working with residents of the Betances Houses, and ended up with the most arrests in his housing command, Inspector O’Sullivan said. Chief Boyce said that the silos in which gang and narcotics detectives used to work made responding to complaints more difficult, but that the recent restructuring would remove those obstacles. “No one should live like Mr. Fernandez lived, with people dealing drugs outside of his apartment,” he said. Mr. Fernandez’s complaints did not spur any arrests, but two men from the hallway were caught separately this year in shootings. One of them, whom Mr. Fernandez named in a police report, was charged this summer with hitting an officer with a metal folding chair and firing three gunshots into a crowd, court papers say. He is being held on Rikers Island on an attempted murder charge. That was too late for Mr. Fernandez. By May, he had moved his family away.'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "papers[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TEXT WRANGLING (NORMALIZING TEXT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We perform some basic text wrangling or preprocessing before diving into topic modeling. We keep things simple here and perform tokenization, lemmatizing nouns, and removing stopwords and any terms having a single character.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "import nltk\n",
    "\n",
    "stop_words = nltk.corpus.stopwords.words('english')\n",
    "wtk = nltk.tokenize.RegexpTokenizer(r'\\w+')\n",
    "wnl = nltk.stem.wordnet.WordNetLemmatizer()\n",
    "\n",
    "def normalise_corpus(papers, full_df):\n",
    "    norm_papers = []\n",
    "    pre_papers = []\n",
    "    drop_index = []\n",
    "    for i in range(len(papers)):\n",
    "        paper = papers[i]\n",
    "\n",
    "        paper = paper.lower()\n",
    "        paper_tokens = [token.strip() for token in wtk.tokenize(paper)]\n",
    "        paper_tokens = [wnl.lemmatize(token) for token in paper_tokens if not token.isnumeric()]\n",
    "        paper_tokens = [token for token in paper_tokens if len(token) > 1]\n",
    "        paper_tokens = [token for token in paper_tokens if token not in stop_words]\n",
    "        paper_tokens = list(filter(None, paper_tokens))\n",
    "\n",
    "        if paper_tokens:\n",
    "            norm_papers.append(paper_tokens)\n",
    "            pre_papers.append(paper)\n",
    "        else:\n",
    "            drop_index.append(i)\n",
    "\n",
    "    pre_df = full_df.drop(full_df.index[drop_index])\n",
    "    \n",
    "    return norm_papers, pre_papers, pre_df\n",
    "\n",
    "# we have pre_papers and pre_titles because the normalizing function removes empty papers and titles\n",
    "# so for consistency the papers and titles that we perform LDA on will be kept "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "49933\n",
      "CPU times: user 2min 27s, sys: 1.13 s, total: 2min 28s\n",
      "Wall time: 2min 28s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "import nltk\n",
    "\n",
    "stop_words = nltk.corpus.stopwords.words('english')\n",
    "wtk = nltk.tokenize.RegexpTokenizer(r'\\w+')\n",
    "wnl = nltk.stem.wordnet.WordNetLemmatizer()\n",
    "\n",
    "def normalise_corpus(papers, titles):\n",
    "    norm_papers = []\n",
    "    pre_papers = []\n",
    "    pre_titles = []\n",
    "    for i in range(len(papers)):\n",
    "        paper = papers[i]\n",
    "        title = titles[i]\n",
    "\n",
    "        paper = paper.lower()\n",
    "        paper_tokens = [token.strip() for token in wtk.tokenize(paper)]\n",
    "        paper_tokens = [wnl.lemmatize(token) for token in paper_tokens if not token.isnumeric()]\n",
    "        paper_tokens = [token for token in paper_tokens if len(token) > 1]\n",
    "        paper_tokens = [token for token in paper_tokens if token not in stop_words]\n",
    "        paper_tokens = list(filter(None, paper_tokens))\n",
    "\n",
    "        if paper_tokens:\n",
    "            norm_papers.append(paper_tokens)\n",
    "            pre_papers.append(paper)\n",
    "            pre_titles.append(title)\n",
    "\n",
    "    return norm_papers, pre_papers, pre_titles\n",
    "\n",
    "# we have pre_papers and pre_titles because the normalizing function removes empty papers and titles\n",
    "# so for consistency the papers and titles that we perform LDA on will be kept \n",
    "\n",
    "norm_papers, pre_papers, pre_titles = normalise_corpus(papers, titles)\n",
    "\n",
    "len(norm_papers), len(pre_papers), len(pre_titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(49933, 49933)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pre_papers), len(pre_titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pre_df.to_csv('data/pre_df.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TEXT REPRESENTATION WITH FEATURE ENGINEERING\n",
    "Before feature engineering and vectorization, we want to extract some useful bi-gram based phrases from our research papers and remove some unnecessary terms. We leverage the very useful gensim.models.Phrases class for this. This capability helps us automatically detect common phrases from a stream of sentences, which are typically multi-word expressions/word n-grams. This implementation draws inspiration from the famous paper by Mikolov, et al., “Distributed Representations of Words and Phrases and their Compositionality,” which you can check out at https://arxiv.org/abs/1310.4546 . We start by extracting and generating words and bi-grams as phrases for each tokenized research paper. We can build this phrase generation model easily with the following code and test it on a sample paper.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['washington', 'congressional', 'republican', 'new', 'fear', 'come', 'health_care', 'lawsuit', 'obama_administration', 'might', 'win', 'incoming', 'trump', 'administration', 'could', 'choose', 'longer', 'defend', 'executive_branch', 'suit', 'challenge', 'administration', 'authority', 'spend', 'billion_dollar', 'health_insurance', 'subsidy', 'american', 'handing', 'house', 'republican', 'big', 'victory', 'issue', 'sudden', 'loss', 'disputed', 'subsidy', 'could_conceivably', 'cause', 'health_care', 'program', 'implode', 'leaving', 'million', 'people', 'without', 'access', 'health_insurance', 'republican']\n"
     ]
    }
   ],
   "source": [
    "import gensim\n",
    "\n",
    "# higher threshold fewer phrases\n",
    "bigram = gensim.models.Phrases(norm_papers, min_count=20, threshold=20, delimiter=b'_')\n",
    "\n",
    "bigram_model = gensim.models.phrases.Phraser(bigram)\n",
    "\n",
    "# sample demonstration\n",
    "\n",
    "print(bigram_model[norm_papers[0]][:50])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can clearly see that we have single words as well as bi-grams (two words separated by an underscore), which tells us that our model works. We leverage the min_count parameter , which tells us that our model ignores all words and bi-grams with total collected count lower than 20 across the corpus (of the input paper as a list of tokenized sentences). \n",
    "\n",
    "\n",
    "Let’s generate phrases for all our tokenized research papers and build a vocabulary that will help us obtain a unique term/phrase to number mapping (since machine or deep learning only works on numeric tensors)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample word to number mappings: [(0, 'access'), (1, 'acknowledge'), (2, 'act'), (3, 'administration'), (4, 'advocate'), (5, 'affordable_care'), (6, 'ally'), (7, 'american'), (8, 'angering'), (9, 'annual'), (10, 'another'), (11, 'anticipated'), (12, 'anticipating'), (13, 'appeal'), (14, 'appealed')]\n",
      "\n",
      " Total Vocabulary Size: 170226\n"
     ]
    }
   ],
   "source": [
    "norm_corpus_bigrams = [bigram_model[doc] for doc in norm_papers]\n",
    "\n",
    "# Create a dictionary representation of the documents.\n",
    "dictionary = gensim.corpora.Dictionary(norm_corpus_bigrams)\n",
    "\n",
    "print('Sample word to number mappings:', list(dictionary.items())[:15])\n",
    "print('\\n Total Vocabulary Size:', len(dictionary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Vocabulary Size: 33588\n"
     ]
    }
   ],
   "source": [
    "# Filter out words that occur less than 20 documents, or more than 50% of the documents.\n",
    "dictionary.filter_extremes(no_below=20, no_above=0.6)\n",
    "print('Total Vocabulary Size:', len(dictionary))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 1), (4, 1), (6, 1), (10, 3), (21, 2), (31, 18), (32, 1), (34, 1), (39, 4), (40, 1), (45, 1), (57, 5), (59, 2), (64, 1), (66, 1), (68, 1), (81, 1), (84, 1), (85, 3), (88, 2), (90, 2), (93, 3), (97, 2), (99, 3), (100, 3), (102, 1), (109, 1), (110, 1), (112, 3), (116, 9), (124, 1), (136, 1), (138, 1), (140, 2), (141, 6), (142, 2), (143, 1), (148, 1), (157, 4), (159, 2), (160, 2), (162, 2), (163, 4), (165, 22), (167, 3), (168, 4), (171, 4), (177, 4), (180, 1), (181, 1)]\n"
     ]
    }
   ],
   "source": [
    "# Transforming corpus into bag of words vectors\n",
    "bow_corpus = [dictionary.doc2bow(text) for text in norm_corpus_bigrams]\n",
    "print(bow_corpus[1][:50])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('access', 1), ('advocate', 1), ('ally', 1), ('another', 3), ('authority', 2), ('case', 18), ('cause', 1), ('challenge', 1), ('come', 4), ('comment', 1), ('confident', 1), ('could', 5), ('court', 2), ('decision', 1), ('defend', 1), ('desperate', 1), ('drop', 1), ('effect', 1), ('effort', 3), ('end', 2), ('era', 2), ('even_though', 3), ('fear', 2), ('fight', 3), ('find', 3), ('found', 1), ('given', 1), ('government', 1), ('handle', 3), ('house', 9), ('inclined', 1), ('judge', 1), ('lack', 1), ('late', 2), ('later', 6), ('law', 2), ('lawsuit', 1), ('leaving', 1), ('many', 4), ('may', 2), ('might', 2), ('money', 2), ('month', 4), ('mr', 22), ('never', 3), ('new', 4), ('office', 4), ('part', 4), ('paying', 1), ('payment', 1)]\n"
     ]
    }
   ],
   "source": [
    "# viewing actual terms and their counts\n",
    "print([(dictionary[idx] , freq) for idx, freq in bow_corpus[1][:50]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### save dictionary, norm_corpus_bigrams, pre_titles, pre_papers, etc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"lists/norm_corpus_bigrams.txt\", \"wb\") as fp:\n",
    "    pickle.dump(norm_corpus_bigrams, fp)  \n",
    "\n",
    "with open(\"lists/pre_titles.txt\", \"wb\") as fp:\n",
    "    pickle.dump(pre_titles, fp)    \n",
    "    \n",
    "with open(\"lists/pre_papers.txt\", \"wb\") as fp:\n",
    "    pickle.dump(pre_papers, fp)    \n",
    "    \n",
    "with open(\"lists/bow_corpus.txt\", \"wb\") as fp:\n",
    "    pickle.dump(bow_corpus, fp)  \n",
    "\n",
    "with open(\"lists/norm_papers.txt\", \"wb\") as fp:\n",
    "    pickle.dump(norm_papers, fp)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary.save('models/dictionary.gensim')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<gensim.models.phrases.Phraser at 0x7fc09820af90>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigram_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "bigram_model.save('models/bigram_model.gensim')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "jupytext": {
   "encoding": "# -*- coding: utf-8 -*-",
   "formats": "ipynb,py:light"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}