BUG: reset setitem_copy on object enlargement

TST: eliminate SettingWithCopyWarnings in tests (catch them)

TST: tests for GH5597

BUG: don't set copy on equal indexes after an operation

Author: jreback

pandas/core/frame.py

@@ -1563,7 +1563,7 @@ def _ixs(self, i, axis=0, copy=False):
 
                     # a ___location index by definition
                     i = _maybe_convert_indices(i, len(self._get_axis(axis)))
-                    return self.reindex(i, takeable=True)
+                    return self.reindex(i, takeable=True)._setitem_copy(True)
                 else:
                     new_values, copy = self._data.fast_2d_xs(i, copy=copy)
                     return Series(new_values, index=self.columns,
@@ -2714,7 +2714,7 @@ def trans(v):
 
             self._clear_item_cache()
         else:
-            return self.take(indexer, axis=axis, convert=False)
+            return self.take(indexer, axis=axis, convert=False, is_copy=False)
 
     def sortlevel(self, level=0, axis=0, ascending=True, inplace=False):
         """
@@ -2760,7 +2760,7 @@ def sortlevel(self, level=0, axis=0, ascending=True, inplace=False):
 
             self._clear_item_cache()
         else:
-            return self.take(indexer, axis=axis, convert=False)
+            return self.take(indexer, axis=axis, convert=False, is_copy=False)
 
     def swaplevel(self, i, j, axis=0):
         """

pandas/core/generic.py

@@ -1064,7 +1064,7 @@ def __delitem__(self, key):
         except KeyError:
             pass
 
-    def take(self, indices, axis=0, convert=True):
+    def take(self, indices, axis=0, convert=True, is_copy=True):
         """
         Analogous to ndarray.take
 
@@ -1073,6 +1073,7 @@ def take(self, indices, axis=0, convert=True):
         indices : list / array of ints
         axis : int, default 0
         convert : translate neg to pos indices (default)
+        is_copy : mark the returned frame as a copy
 
         Returns
         -------
@@ -1090,12 +1091,17 @@ def take(self, indices, axis=0, convert=True):
             labels = self._get_axis(axis)
             new_items = labels.take(indices)
             new_data = self._data.reindex_axis(new_items, indexer=indices,
-                                               axis=0)
+                                               axis=baxis)
         else:
-            new_data = self._data.take(indices, axis=baxis, verify=convert)
-        return self._constructor(new_data)\
-                   ._setitem_copy(True)\
-                   .__finalize__(self)
+            new_data = self._data.take(indices, axis=baxis)
+
+        result = self._constructor(new_data).__finalize__(self)
+
+        # maybe set copy if we didn't actually change the index
+        if is_copy and not result._get_axis(axis).equals(self._get_axis(axis)):
+            result = result._setitem_copy(is_copy)
+
+        return result
 
     # TODO: Check if this was clearer in 0.12
     def select(self, crit, axis=0):

pandas/core/indexing.py

@@ -209,6 +209,7 @@ def _setitem_with_indexer(self, indexer, value):
                     labels = _safe_append_to_index(index, key)
                     self.obj._data = self.obj.reindex_axis(labels, i)._data
                     self.obj._maybe_update_cacher(clear=True)
+                    self.obj._setitem_copy(False)
 
                     if isinstance(labels, MultiIndex):
                         self.obj.sortlevel(inplace=True)

pandas/tests/test_index.py

@@ -1381,8 +1381,10 @@ def test_set_value_keeps_names(self):
             columns=['one', 'two', 'three', 'four'],
             index=idx)
         df = df.sortlevel()
+        self.assert_(df._is_copy is False)
         self.assertEqual(df.index.names, ('Name', 'Number'))
         df = df.set_value(('grethe', '4'), 'one', 99.34)
+        self.assert_(df._is_copy is False)
         self.assertEqual(df.index.names, ('Name', 'Number'))
 
     def test_names(self):

pandas/tests/test_indexing.py

@@ -1049,6 +1049,8 @@ def f(name,df2):
             return Series(np.arange(df2.shape[0]),name=df2.index.values[0]).reindex(f_index)
         new_df = pd.concat([ f(name,df2) for name, df2 in grp ],axis=1).T
 
+        # we are actually operating on a copy here
+        # but in this case, that's ok
         for name, df2 in grp:
             new_vals = np.arange(df2.shape[0])
             df.ix[name, 'new_col'] = new_vals
@@ -1769,7 +1771,8 @@ def f():
                               'c' : [42,42,2,3,4,42,6]})
 
         def f():
-            df[df.a.str.startswith('o')]['c'] = 42
+            indexer = df.a.str.startswith('o')
+            df[indexer]['c'] = 42
         self.assertRaises(com.SettingWithCopyError, f)
         df['c'][df.a.str.startswith('o')] = 42
         assert_frame_equal(df,expected)
@@ -1785,7 +1788,8 @@ def f():
         # warnings
         pd.set_option('chained_assignment','warn')
         df = DataFrame({'A':['aaa','bbb','ccc'],'B':[1,2,3]})
-        df.loc[0]['A'] = 111
+        with tm.assert_produces_warning(expected_warning=com.SettingWithCopyWarning):
+            df.loc[0]['A'] = 111
 
         # make sure that _is_copy is picked up reconstruction
         # GH5475
@@ -1797,6 +1801,55 @@ def f():
             df2["B"] = df2["A"]
             df2["B"] = df2["A"]
 
+        # a suprious raise as we are setting the entire column here
+        # GH5597
+        pd.set_option('chained_assignment','raise')
+        from string import ascii_letters as letters
+
+        def random_text(nobs=100):
+            df = []
+            for i in range(nobs):
+                idx= np.random.randint(len(letters), size=2)
+                idx.sort()
+                df.append([letters[idx[0]:idx[1]]])
+
+            return DataFrame(df, columns=['letters'])
+
+        df = random_text(100000)
+
+        # always a copy
+        x = df.iloc[[0,1,2]]
+        self.assert_(x._is_copy is True)
+        x = df.iloc[[0,1,2,4]]
+        self.assert_(x._is_copy is True)
+
+        # explicity copy
+        indexer = df.letters.apply(lambda x : len(x) > 10)
+        df = df.ix[indexer].copy()
+        self.assert_(df._is_copy is False)
+        df['letters'] = df['letters'].apply(str.lower)
+
+        # implicity take
+        df = random_text(100000)
+        indexer = df.letters.apply(lambda x : len(x) > 10)
+        df = df.ix[indexer]
+        self.assert_(df._is_copy is True)
+        df.loc[:,'letters'] = df['letters'].apply(str.lower)
+
+        # this will raise
+        #df['letters'] = df['letters'].apply(str.lower)
+
+        df = random_text(100000)
+        indexer = df.letters.apply(lambda x : len(x) > 10)
+        df.ix[indexer,'letters'] = df.ix[indexer,'letters'].apply(str.lower)
+
+        # an identical take, so no copy
+        df = DataFrame({'a' : [1]}).dropna()
+        self.assert_(df._is_copy is False)
+        df['a'] += 1
+
+        pd.set_option('chained_assignment','warn')
+
     def test_float64index_slicing_bug(self):
         # GH 5557, related to slicing a float index
         ser = {256: 2321.0, 1: 78.0, 2: 2716.0, 3: 0.0, 4: 369.0, 5: 0.0, 6: 269.0, 7: 0.0, 8: 0.0, 9: 0.0, 10: 3536.0, 11: 0.0, 12: 24.0, 13: 0.0, 14: 931.0, 15: 0.0, 16: 101.0, 17: 78.0, 18: 9643.0, 19: 0.0, 20: 0.0, 21: 0.0, 22: 63761.0, 23: 0.0, 24: 446.0, 25: 0.0, 26: 34773.0, 27: 0.0, 28: 729.0, 29: 78.0, 30: 0.0, 31: 0.0, 32: 3374.0, 33: 0.0, 34: 1391.0, 35: 0.0, 36: 361.0, 37: 0.0, 38: 61808.0, 39: 0.0, 40: 0.0, 41: 0.0, 42: 6677.0, 43: 0.0, 44: 802.0, 45: 0.0, 46: 2691.0, 47: 0.0, 48: 3582.0, 49: 0.0, 50: 734.0, 51: 0.0, 52: 627.0, 53: 70.0, 54: 2584.0, 55: 0.0, 56: 324.0, 57: 0.0, 58: 605.0, 59: 0.0, 60: 0.0, 61: 0.0, 62: 3989.0, 63: 10.0, 64: 42.0, 65: 0.0, 66: 904.0, 67: 0.0, 68: 88.0, 69: 70.0, 70: 8172.0, 71: 0.0, 72: 0.0, 73: 0.0, 74: 64902.0, 75: 0.0, 76: 347.0, 77: 0.0, 78: 36605.0, 79: 0.0, 80: 379.0, 81: 70.0, 82: 0.0, 83: 0.0, 84: 3001.0, 85: 0.0, 86: 1630.0, 87: 7.0, 88: 364.0, 89: 0.0, 90: 67404.0, 91: 9.0, 92: 0.0, 93: 0.0, 94: 7685.0, 95: 0.0, 96: 1017.0, 97: 0.0, 98: 2831.0, 99: 0.0, 100: 2963.0, 101: 0.0, 102: 854.0, 103: 0.0, 104: 0.0, 105: 0.0, 106: 0.0, 107: 0.0, 108: 0.0, 109: 0.0, 110: 0.0, 111: 0.0, 112: 0.0, 113: 0.0, 114: 0.0, 115: 0.0, 116: 0.0, 117: 0.0, 118: 0.0, 119: 0.0, 120: 0.0, 121: 0.0, 122: 0.0, 123: 0.0, 124: 0.0, 125: 0.0, 126: 67744.0, 127: 22.0, 128: 264.0, 129: 0.0, 260: 197.0, 268: 0.0, 265: 0.0, 269: 0.0, 261: 0.0, 266: 1198.0, 267: 0.0, 262: 2629.0, 258: 775.0, 257: 0.0, 263: 0.0, 259: 0.0, 264: 163.0, 250: 10326.0, 251: 0.0, 252: 1228.0, 253: 0.0, 254: 2769.0, 255: 0.0}